diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 04663a22..00000000 --- a/.drone.yml +++ /dev/null @@ -1,290 +0,0 @@ ---- -kind: pipeline -name: default - -node: - nix-daemon: 1 - -steps: - - name: check formatting - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr rust --run "cargo fmt -- --check" - - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - - name: unit + func tests - image: nixpkgs/nix:nixos-22.05 - environment: - GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage - GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-build --no-build-output --attr test.amd64 - - ./result/bin/garage_db-* - - ./result/bin/garage_api-* - - ./result/bin/garage_model-* - - ./result/bin/garage_rpc-* - - ./result/bin/garage_table-* - - ./result/bin/garage_util-* - - ./result/bin/garage_web-* - - ./result/bin/garage-* - - ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false) - - rm result - - rm -rv tmp-garage-integration - - - name: integration tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - -trigger: - event: - - custom - - push - - pull_request - - tag - - cron - ---- -kind: pipeline -type: docker -name: release-linux-amd64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "x86_64-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/amd64" - CONTAINER_NAME: "dxflrs/amd64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-i386 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "i686-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/386" - CONTAINER_NAME: "dxflrs/386_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "aarch64-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm64" - CONTAINER_NAME: "dxflrs/arm64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "armv6l-unknown-linux-musleabihf" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm" - CONTAINER_NAME: "dxflrs/arm_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: refresh-release-page - -node: - nix-daemon: 1 - -steps: - - name: multiarch-docker - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - HOME: "/root" - commands: - - mkdir -p /root/.docker - - echo $DOCKER_AUTH > /root/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "multiarch_docker" - - name: refresh-index - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - commands: - - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --attr release --run "refresh_index" - -depends_on: - - release-linux-amd64 - - release-linux-i386 - - release-linux-arm64 - - release-linux-arm - -trigger: - event: - - promote - - cron - ---- -kind: signature -hmac: 24aebbcdba84fd0cdf963061d7bb72ae5b915bfdd0f50c7b019001126fb7fa56 - -... diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml new file mode 100644 index 00000000..4dc7d3c9 --- /dev/null +++ b/.woodpecker/debug.yaml @@ -0,0 +1,44 @@ +labels: + nix: "enabled" + +when: + event: + - push + - tag + - pull_request + - deployment + - cron + - manual + +steps: + - name: check formatting + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.fmt + + - name: build + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.dev + + - name: unit + func tests (lmdb) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-lmdb + + - name: unit + func tests (sqlite) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-sqlite + + - name: unit + func tests (fjall) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-fjall + + - name: integration tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.dev + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + depends_on: [ build ] diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml new file mode 100644 index 00000000..8f3b482f --- /dev/null +++ b/.woodpecker/publish.yaml @@ -0,0 +1,33 @@ +labels: + nix: "enabled" + +when: + event: + - deployment + - cron + +depends_on: + - release + +steps: + - name: refresh-index + image: nixpkgs/nix:nixos-24.05 + environment: + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf + - nix-shell --attr ci --run "refresh_index" + + - name: multiarch-docker + image: nixpkgs/nix:nixos-24.05 + environment: + DOCKER_AUTH: + from_secret: docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "multiarch_docker" diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml new file mode 100644 index 00000000..a94a9ccf --- /dev/null +++ b/.woodpecker/release.yaml @@ -0,0 +1,71 @@ +labels: + nix: "enabled" + +when: + event: + - deployment + - cron + +matrix: + include: + - ARCH: amd64 + TARGET: x86_64-unknown-linux-musl + - ARCH: i386 + TARGET: i686-unknown-linux-musl + - ARCH: arm64 + TARGET: aarch64-unknown-linux-musl + - ARCH: arm + TARGET: armv6l-unknown-linux-musleabihf + +steps: + - name: build + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + + - name: check is static binary + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage" + + - name: integration tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + when: + - matrix: + ARCH: amd64 + - matrix: + ARCH: i386 + + - name: upgrade tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) + when: + - matrix: + ARCH: amd64 + + - name: push static binary + image: nixpkgs/nix:nixos-24.05 + environment: + TARGET: "${TARGET}" + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - nix-shell --attr ci --run "to_s3" + + - name: docker build and publish + image: nixpkgs/nix:nixos-24.05 + environment: + DOCKER_PLATFORM: "linux/${ARCH}" + CONTAINER_NAME: "dxflrs/${ARCH}_garage" + DOCKER_AUTH: + from_secret: docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "to_docker" diff --git a/Cargo.lock b/Cargo.lock index 873cbce4..7473d9af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,54 +1,84 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.3.4", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" @@ -61,63 +91,80 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", - "windows-sys", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] [[package]] name = "arrayvec" @@ -137,23 +184,21 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.1" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" dependencies = [ - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "zstd", - "zstd-safe", ] [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -162,255 +207,180 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" -version = "1.1.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "aws-config" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand 1.9.0", - "hex", - "http", - "hyper", - "ring", - "time 0.3.28", - "tokio", - "tower", - "tracing", - "zeroize", -] +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-credential-types" -version = "0.55.3" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand 1.9.0", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-endpoint" -version = "0.55.3" +name = "aws-runtime" +version = "1.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +checksum = "959dab27ce613e6c9658eb3621064d0e2027e5f2acb65bc526a43577facea557" dependencies = [ "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", - "http-body", - "lazy_static", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-config" +version = "1.99.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e62e5ffb669e13f084c4e1d89d687604e001187f61503606a7f8cc7a411995" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] name = "aws-sdk-s3" -version = "0.28.0" +version = "1.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f" +checksum = "06673901e961f20fa8d7da907da48f7ad6c1b383e3726c22bd418900f015abe1" dependencies = [ "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", - "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", - "http", - "http-body", - "once_cell", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "lru", "percent-encoding", - "regex", - "tokio-stream", - "tower", + "regex-lite", + "sha2", "tracing", "url", ] -[[package]] -name = "aws-sdk-sso" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "regex", - "tokio-stream", - "tower", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", - "tracing", -] - [[package]] name = "aws-sigv4" -version = "0.55.3" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", "form_urlencoded", "hex", "hmac", - "http", - "once_cell", + "http 0.2.12", + "http 1.4.0", "percent-encoding", - "regex", "sha2", - "time 0.3.28", + "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "0.55.3" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.55.3" +version = "0.63.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6" +checksum = "23374b9170cbbcc6f5df8dc5ebb9b6c5c28a3c8f599f0e8b8b10eb6f4a5c6e74" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", - "crc32c", - "crc32fast", + "crc-fast", "hex", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "md-5", "pin-project-lite", "sha1", @@ -418,35 +388,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 1.9.0", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "pin-project-lite", - "rustls 0.20.8", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.55.3" +version = "0.60.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8" +checksum = "dc12f8b310e38cad85cf3bef45ad236f470717393c613266ce0a89512286b650" dependencies = [ "aws-smithy-types", "bytes", @@ -455,96 +401,152 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.55.3" +version = "0.62.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", - "http", - "http-body", - "hyper", - "once_cell", + "futures-util", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util 0.7.8", "tracing", ] [[package]] -name = "aws-smithy-http-tower" -version = "0.55.3" +name = "aws-smithy-http-client" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" dependencies = [ - "aws-smithy-http", + "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "bytes", - "http", - "http-body", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "pin-project-lite", - "tower", + "rustls 0.21.12", + "rustls-native-certs 0.8.3", + "tokio", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.55.3" +version = "0.61.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" dependencies = [ "aws-smithy-types", ] [[package]] -name = "aws-smithy-query" -version = "0.55.3" +name = "aws-smithy-observability" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +checksum = "ef1fcbefc7ece1d70dcce29e490f269695dfca2d2bacdeaf9e5c3f799e4e6a42" dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5b6167fcdf47399024e81ac08e795180c576a20e4d4ce67949f9a88ae37dc1" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", "aws-smithy-types", - "urlencoding", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efce7aaaf59ad53c5412f14fc19b2d5c6ab2c3ec688d272fd31f76ec12f44fb0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.55.3" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +checksum = "65f172bcb02424eb94425db8aed1b6d583b5104d4d5ddddf22402c661a320048" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", - "time 0.3.28", + "serde", + "time", + "tokio", + "tokio-util 0.7.18", ] [[package]] name = "aws-smithy-xml" -version = "0.55.3" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.3" +version = "1.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "http", "rustc_version", "tracing", ] @@ -555,24 +557,24 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "getrandom", + "getrandom 0.2.17", "instant", "rand", ] [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-link", ] [[package]] @@ -583,9 +585,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.3" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64-simd" @@ -597,6 +605,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bincode" version = "1.3.3" @@ -614,9 +628,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "blake2" @@ -638,33 +652,33 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "bytes-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", "either", @@ -672,25 +686,33 @@ dependencies = [ [[package]] name = "bytesize" -version = "1.3.0" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" + +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" dependencies = [ + "find-msvc-tools", "jobserver", "libc", + "shlex", ] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -699,19 +721,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] -name = "chrono" -version = "0.4.26" +name = "cfg_aliases" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", - "winapi", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", ] [[package]] @@ -727,20 +763,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.0" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.0" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstream", "anstyle", @@ -750,33 +785,66 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "compression-core", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -784,94 +852,132 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] -name = "crc32c" -version = "0.6.4" +name = "crc" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest", + "rustversion", + "spin 0.10.0", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ - "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core", "typenum", ] [[package]] -name = "darling" -version = "0.14.4" +name = "ctr" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -879,27 +985,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -913,10 +1019,27 @@ dependencies = [ ] [[package]] -name = "deranged" -version = "0.3.8" +name = "dashmap" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] [[package]] name = "derivative" @@ -941,52 +1064,60 @@ dependencies = [ ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "cfg-if", - "dirs-sys-next", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "double-ended-peekable" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.9.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -997,50 +1128,25 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "err-derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", - "synstructure", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.61.2", ] [[package]] name = "fallible-iterator" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fallible-streaming-iterator" @@ -1050,18 +1156,15 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "fastrand" -version = "2.0.0" +name = "find-msvc-tools" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "fixedbitset" @@ -1069,6 +1172,23 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fjall" +version = "2.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +dependencies = [ + "byteorder", + "byteview", + "dashmap 6.1.0", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1076,10 +1196,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "form_urlencoded" -version = "1.2.0" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1088,21 +1220,11 @@ dependencies = [ name = "format_table" version = "0.1.1" -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "futures" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1115,9 +1237,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1125,15 +1247,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1142,38 +1264,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1187,35 +1309,30 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "garage" -version = "0.8.4" +version = "1.3.1" dependencies = [ "assert-json-diff", "async-trait", - "aws-config", "aws-sdk-s3", + "aws-smithy-runtime", "backtrace", - "base64 0.21.3", + "base64 0.21.7", "bytes", "bytesize", "chrono", + "crc32fast", "format_table", "futures", - "futures-util", - "garage_api", + "garage_api_admin", + "garage_api_common", + "garage_api_k2v", + "garage_api_s3", "garage_block", "garage_db", "garage_model", + "garage_net", "garage_rpc", "garage_table", "garage_util", @@ -1223,78 +1340,162 @@ dependencies = [ "git-version", "hex", "hmac", - "http", - "hyper", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", "k2v-client", "kuska-sodiumoxide", - "netapp", + "mktemp", "opentelemetry", "opentelemetry-otlp", "opentelemetry-prometheus", "parse_duration", - "prometheus", - "rand", "serde", - "serde_bytes", "serde_json", + "sha1", "sha2", "static_init", "structopt", + "syslog-tracing", "timeago", "tokio", - "toml", "tracing", + "tracing-journald", "tracing-subscriber", ] [[package]] -name = "garage_api" -version = "0.8.4" +name = "garage_api_admin" +version = "1.3.1" dependencies = [ + "argon2", "async-trait", - "base64 0.21.3", - "bytes", - "chrono", - "crypto-common", - "err-derive", - "form_urlencoded", "futures", - "futures-util", - "garage_block", + "garage_api_common", "garage_model", "garage_rpc", "garage_table", "garage_util", "hex", - "hmac", - "http", - "http-range", - "httpdate", - "hyper", - "idna", - "md-5", - "multer", - "nom", + "http 1.4.0", + "hyper 1.8.1", "opentelemetry", "opentelemetry-prometheus", + "prometheus", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_common" +version = "1.3.1" +dependencies = [ + "base64 0.21.7", + "bytes", + "chrono", + "crc32c", + "crc32fast", + "crypto-common", + "futures", + "garage_model", + "garage_table", + "garage_util", + "hex", + "hmac", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "md-5", + "nom", + "opentelemetry", + "pin-project", + "serde", + "serde_json", + "sha1", + "sha2", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_k2v" +version = "1.3.1" +dependencies = [ + "base64 0.21.7", + "futures", + "garage_api_common", + "garage_model", + "garage_table", + "garage_util", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "opentelemetry", + "percent-encoding", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_s3" +version = "1.3.1" +dependencies = [ + "aes-gcm", + "async-compression", + "base64 0.21.7", + "bytes", + "chrono", + "crc32c", + "crc32fast", + "form_urlencoded", + "futures", + "garage_api_common", + "garage_block", + "garage_model", + "garage_net", + "garage_rpc", + "garage_table", + "garage_util", + "hex", + "http 1.4.0", + "http-body-util", + "http-range", + "httpdate", + "hyper 1.8.1", + "md-5", + "multer", + "opentelemetry", "percent-encoding", "pin-project", - "prometheus", "quick-xml", "roxmltree", "serde", - "serde_bytes", "serde_json", + "sha1", "sha2", + "thiserror 2.0.18", "tokio", "tokio-stream", + "tokio-util 0.7.18", "tracing", "url", ] [[package]] name = "garage_block" -version = "0.8.4" +version = "1.3.1" dependencies = [ "arc-swap", "async-compression", @@ -1302,8 +1503,8 @@ dependencies = [ "bytes", "bytesize", "futures", - "futures-util", "garage_db", + "garage_net", "garage_rpc", "garage_table", "garage_util", @@ -1311,77 +1512,97 @@ dependencies = [ "opentelemetry", "rand", "serde", - "serde_bytes", "tokio", - "tokio-util 0.7.8", + "tokio-util 0.7.18", "tracing", "zstd", ] [[package]] name = "garage_db" -version = "0.8.4" +version = "1.3.1" dependencies = [ - "clap 4.4.0", - "err-derive", + "fjall", "heed", - "hexdump", "mktemp", - "pretty_env_logger", + "parking_lot", + "r2d2", + "r2d2_sqlite", "rusqlite", - "sled", + "thiserror 2.0.18", "tracing", ] [[package]] name = "garage_model" -version = "0.8.4" +version = "1.3.1" dependencies = [ - "arc-swap", "async-trait", - "base64 0.21.3", + "base64 0.21.7", "blake2", "chrono", - "err-derive", "futures", - "futures-util", "garage_block", "garage_db", + "garage_net", "garage_rpc", "garage_table", "garage_util", "hex", - "netapp", - "opentelemetry", + "http 1.4.0", + "parse_duration", "rand", "serde", "serde_bytes", + "thiserror 2.0.18", "tokio", "tracing", "zstd", ] +[[package]] +name = "garage_net" +version = "1.3.1" +dependencies = [ + "arc-swap", + "bytes", + "cfg-if", + "futures", + "hex", + "kuska-handshake", + "kuska-sodiumoxide", + "log", + "opentelemetry", + "opentelemetry-contrib", + "pin-project", + "pretty_env_logger", + "rand", + "rmp-serde", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util 0.7.18", +] + [[package]] name = "garage_rpc" -version = "0.8.4" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", - "bytes", "bytesize", - "err-derive", "format_table", "futures", - "futures-util", - "garage_db", + "garage_net", "garage_util", "gethostname", "hex", - "itertools 0.10.5", + "ipnet", + "itertools 0.12.1", "k8s-openapi", "kube", "kuska-sodiumoxide", - "netapp", "nix", "opentelemetry", "pnet_datalink", @@ -1391,18 +1612,17 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "thiserror 2.0.18", "tokio", - "tokio-stream", "tracing", ] [[package]] name = "garage_table" -version = "0.8.4" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", - "bytes", "futures", "futures-util", "garage_db", @@ -1420,25 +1640,22 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.8.4" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", "blake2", - "bytes", "bytesize", "chrono", - "digest", - "err-derive", "futures", "garage_db", + "garage_net", "hex", "hexdump", - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "lazy_static", "mktemp", - "netapp", "opentelemetry", "rand", "rmp-serde", @@ -1446,6 +1663,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "thiserror 2.0.18", "tokio", "toml", "tracing", @@ -1454,18 +1672,20 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.8.4" +version = "1.3.1" dependencies = [ - "err-derive", - "futures", - "garage_api", + "garage_api_common", + "garage_api_s3", "garage_model", "garage_table", "garage_util", - "http", - "hyper", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", "opentelemetry", "percent-encoding", + "thiserror 2.0.18", + "tokio", "tracing", ] @@ -1486,64 +1706,109 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" dependencies = [ "libc", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", ] [[package]] name = "gimli" -version = "0.28.0" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "git-version" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" dependencies = [ "git-version-macro", - "proc-macro-hack", ] [[package]] name = "git-version-macro" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ - "proc-macro-hack", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] -name = "h2" -version = "0.3.21" +name = "guardian" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap 1.9.3", + "http 0.2.12", + "indexmap 2.13.0", "slab", "tokio", - "tokio-util 0.7.8", + "tokio-util 0.7.18", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util 0.7.18", "tracing", ] @@ -1555,21 +1820,41 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", ] [[package]] -name = "hashlink" -version = "0.8.3" +name = "hashbrown" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "hashbrown 0.14.0", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", ] [[package]] @@ -1583,9 +1868,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" @@ -1627,9 +1912,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1639,12 +1924,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hexdump" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2" +checksum = "cf31ab66ed8145a1c7427bd8e9b42a6131bd74ccf444f69b9e620c2e73ded832" dependencies = [ "arrayvec", - "itertools 0.4.19", ] [[package]] @@ -1657,10 +1941,19 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.9" +name = "home" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1668,13 +1961,46 @@ dependencies = [ ] [[package]] -name = "http-body" -version = "0.4.5" +name = "http" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "http", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1692,9 +2018,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -1704,28 +2030,28 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1733,60 +2059,114 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.23.2" +name = "hyper" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.8", - "rustls-native-certs", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", "tokio", - "tokio-rustls 0.23.4", + "want", ] [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.32", "log", - "rustls 0.21.6", - "rustls-native-certs", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.4.0", + "hyper 1.8.1", + "hyper-util", + "rustls 0.22.4", + "rustls-native-certs 0.7.3", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", ] [[package]] -name = "iana-time-zone" -version = "0.1.57" +name = "hyper-util" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.2", + "system-configuration 0.6.1", + "tokio", + "tower-layer", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1798,6 +2178,87 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1806,12 +2267,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1826,28 +2298,46 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.16.1", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", ] [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] [[package]] -name = "ipnet" -version = "2.8.0" +name = "interval-heap" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" +dependencies = [ + "compare", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "ipnetwork" @@ -1860,20 +2350,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", - "rustix", - "windows-sys", + "libc", + "windows-sys 0.61.2", ] [[package]] -name = "itertools" -version = "0.4.19" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -1885,96 +2375,107 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.9" +name = "itertools" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "json-patch" -version = "0.2.7" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +checksum = "ec9ad60d674508f3ca8f380a928cfe7b096bc729c4e2dbfe3852bc45da3ab30b" dependencies = [ "serde", "serde_json", - "treediff", + "thiserror 1.0.69", ] [[package]] -name = "jsonpath_lib" -version = "0.3.0" +name = "jsonpath-rust" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +checksum = "96acbc6188d3bd83519d053efec756aa4419de62ec47be7f28dec297f7dc9eb0" dependencies = [ - "log", - "serde", + "pest", + "pest_derive", + "regex", "serde_json", + "thiserror 1.0.69", ] [[package]] name = "k2v-client" version = "0.0.4" dependencies = [ + "aws-sdk-config", "aws-sigv4", - "base64 0.21.3", - "clap 4.4.0", + "base64 0.21.7", + "clap 4.5.54", "format_table", "hex", - "http", - "hyper", - "hyper-rustls 0.24.1", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.26.0", + "hyper-util", "log", "percent-encoding", "serde", "serde_json", "sha2", - "thiserror", + "thiserror 2.0.18", "tokio", - "tracing", "tracing-subscriber", ] [[package]] name = "k8s-openapi" -version = "0.16.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" +checksum = "550f99d93aa4c2b25de527bce492d772caf5e21d7ac9bd4b508ba781c8d91e30" dependencies = [ - "base64 0.13.1", - "bytes", + "base64 0.21.7", "chrono", - "http", - "percent-encoding", "serde", "serde-value", "serde_json", - "url", ] [[package]] name = "kube" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb19108692aeafebb108fd0a1c381c06ac4c03859652599420975165e939b8a" +checksum = "462fe330a0617b276ec864c2255810adcdf519ecb6844253c54074b2086a97bc" dependencies = [ "k8s-openapi", "kube-client", @@ -1985,35 +2486,35 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e1a80ecd1b1438a2fc004549e155d47250b9e01fbfcf4cfbe9c8b56a085593" +checksum = "7fe0d65dd6f3adba29cfb84f19dfe55449c7f6c35425f9d8294bec40313e0b64" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "bytes", "chrono", - "dirs-next", "either", "futures", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", + "home", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "hyper-timeout", - "jsonpath_lib", + "jsonpath-rust", "k8s-openapi", "kube-core", "pem", "pin-project", - "rustls 0.20.8", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "secrecy", "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-util 0.7.8", + "tokio-util 0.7.18", "tower", "tower-http", "tracing", @@ -2021,56 +2522,58 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4d780f2bb048eeef64a4c6b2582d26a0fe19e30b4d3cc9e081616e1779c5d47" +checksum = "a6b42844e9172f631b8263ea9ce003b9251da13beb1401580937ad206dd82f4c" dependencies = [ "chrono", "form_urlencoded", - "http", + "http 0.2.12", "json-patch", "k8s-openapi", "once_cell", "schemars", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "kube-derive" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98459d53b2841237392cd6959956185b2df15c19d32c3b275ed6ca7b7ee1adae" +checksum = "f5b5a111ee287bd237b8190b8c39543ea9fd22f79e9c32a36c24e08234bcda22" dependencies = [ "darling", "proc-macro2", "quote", "serde_json", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] name = "kube-runtime" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7769af142ee2e46bfa44bd393cf7f40b9d8b80d2e11f6317399551ed17760beb" +checksum = "2bc06275064c81056fbb28ea876b3fb339d970e8132282119359afca0835c0ea" dependencies = [ "ahash", + "async-trait", "backoff", "derivative", "futures", + "hashbrown 0.14.5", "json-patch", "k8s-openapi", "kube-client", - "parking_lot 0.12.1", + "parking_lot", "pin-project", "serde", "serde_json", "smallvec", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-util 0.7.8", + "tokio-util 0.7.18", "tracing", ] @@ -2084,7 +2587,7 @@ dependencies = [ "hex", "kuska-sodiumoxide", "log", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -2100,15 +2603,15 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libsodium-sys" @@ -2124,9 +2627,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.26.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" dependencies = [ "cc", "pkg-config", @@ -2134,16 +2637,22 @@ dependencies = [ ] [[package]] -name = "linked-hash-map" -version = "0.5.6" +name = "linux-raw-sys" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lmdb-rkv-sys" @@ -2158,52 +2667,82 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.20" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "lsm-tree" +version = "2.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "mime" @@ -2219,45 +2758,44 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "wasi", + "windows-sys 0.61.2", ] [[package]] name = "mktemp" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a" +checksum = "69fed8fbcd01affec44ac226784c6476a6006d98d13e33bc0ca7977aaf046bd8" dependencies = [ "uuid", ] [[package]] name = "multer" -version = "2.1.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.4.0", "httparse", - "log", "memchr", "mime", "spin 0.9.8", @@ -2270,41 +2808,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "netapp" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d" -dependencies = [ - "arc-swap", - "async-trait", - "bytes", - "cfg-if", - "err-derive", - "futures", - "hex", - "kuska-handshake", - "kuska-sodiumoxide", - "log", - "opentelemetry", - "opentelemetry-contrib", - "pin-project", - "rand", - "rmp-serde", - "serde", - "tokio", - "tokio-stream", - "tokio-util 0.7.8", -] - [[package]] name = "nix" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.10.0", "cfg-if", + "cfg_aliases 0.2.1", "libc", ] @@ -2324,23 +2836,13 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "overload", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -2379,20 +2881,25 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num-conv" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -2413,18 +2920,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ "hermit-abi", "libc", @@ -2432,24 +2939,42 @@ dependencies = [ [[package]] name = "object" -version = "0.32.0" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" @@ -2459,7 +2984,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" dependencies = [ "async-trait", "crossbeam-channel", - "dashmap", + "dashmap 4.0.2", "fnv", "futures-channel", "futures-executor", @@ -2469,7 +2994,7 @@ dependencies = [ "percent-encoding", "pin-project", "rand", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", ] @@ -2493,10 +3018,10 @@ dependencies = [ "async-trait", "futures", "futures-util", - "http", + "http 0.2.12", "opentelemetry", "prost", - "thiserror", + "thiserror 1.0.69", "tokio", "tonic", "tonic-build", @@ -2515,24 +3040,18 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] [[package]] name = "outref" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "page_size" @@ -2546,50 +3065,25 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", - "windows-targets", + "windows-link", ] [[package]] @@ -2604,61 +3098,128 @@ dependencies = [ ] [[package]] -name = "paste" -version = "1.0.14" +name = "password-hash" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde_core", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pest_meta" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" +dependencies = [ + "pest", + "sha2", +] [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 2.13.0", ] [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2668,24 +3229,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pnet_base" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e" +checksum = "fe4cf6fb3ab38b68d01ab2aea03ed3d1132b4868fa4e06285f29f16da01c5f4c" dependencies = [ "no-std-net", ] [[package]] name = "pnet_datalink" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce" +checksum = "ad5854abf0067ebbd3967f7d45ebc8976ff577ff0c7bd101c4973ae3c70f98fe" dependencies = [ "ipnetwork", "libc", @@ -2696,19 +3257,49 @@ dependencies = [ [[package]] name = "pnet_sys" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16" +checksum = "417c0becd1b573f6d544f73671070b039051e5ad819cc64aa96377b536128d00" dependencies = [ "libc", "winapi", ] [[package]] -name = "ppv-lite86" -version = "0.2.17" +name = "polyval" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_env_logger" @@ -2744,34 +3335,28 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prometheus" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ "cfg-if", "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot", "protobuf", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -2844,14 +3429,52 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.33" +name = "quick_cache" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c" +dependencies = [ + "r2d2", + "rusqlite", + "uuid", +] + [[package]] name = "rand" version = "0.8.5" @@ -2879,98 +3502,69 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.17", ] [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", + "bitflags 2.10.0", ] [[package]] name = "regex" -version = "1.9.4" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.7", - "regex-syntax 0.7.5", + "regex-automata", + "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax", ] [[package]] -name = "regex-syntax" -version = "0.6.29" +name = "regex-lite" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.3", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls 0.24.1", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -2978,11 +3572,13 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.6", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", + "system-configuration 0.5.1", "tokio", "tokio-rustls 0.24.1", "tower-service", @@ -2995,57 +3591,50 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", + "cfg-if", + "getrandom 0.2.17", "libc", - "once_cell", - "spin 0.5.2", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.52.0", ] [[package]] name = "rmp" -version = "0.8.12" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" dependencies = [ - "byteorder", "num-traits", - "paste", ] [[package]] name = "rmp-serde" -version = "0.15.5" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" dependencies = [ - "byteorder", "rmp", "serde", ] [[package]] name = "roxmltree" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f595a457b6b8c6cda66a48503e92ee8d19342f905948f29c383200ec9eb1d8" -dependencies = [ - "xmlparser", -] +checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f" [[package]] name = "rusqlite" -version = "0.29.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.10.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3055,54 +3644,74 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.9" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.10.0", "errno", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", + "rustls-webpki 0.101.7", "sct", - "webpki", ] [[package]] name = "rustls" -version = "0.21.6" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ - "log", "ring", - "rustls-webpki", - "sct", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", ] [[package]] @@ -3111,42 +3720,96 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ - "openssl-probe", - "rustls-pemfile", + "openssl-probe 0.1.6", + "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", ] [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.3", + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", "untrusted", ] [[package]] -name = "rustversion" -version = "1.0.14" +name = "rustls-webpki" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "same-file" @@ -3159,18 +3822,27 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", ] [[package]] name = "schemars" -version = "0.8.12" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", @@ -3180,14 +3852,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.12" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -3198,9 +3870,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", @@ -3218,12 +3890,25 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", - "core-foundation", + "bitflags 2.10.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3231,26 +3916,33 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", ] [[package]] -name = "semver" -version = "1.0.18" +name = "self_cell" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -3266,52 +3958,63 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap 2.0.0", "itoa", - "ryu", + "memchr", "serde", + "serde_core", + "zmij", ] [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -3330,21 +4033,22 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.26" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.13.0", + "itoa", "ryu", "serde", - "yaml-rust", + "unsafe-libyaml", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -3353,9 +4057,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -3364,79 +4068,61 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] -name = "signal-hook-registry" -version = "1.4.1" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] [[package]] name = "slab" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" -version = "1.11.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.4.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", - "winapi", + "windows-sys 0.52.0", ] [[package]] name = "socket2" -version = "0.5.3" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.60.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -3444,27 +4130,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "static_init" -version = "1.0.3" +name = "spin" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2a1c578e98c1c16fc3b8ec1328f7659a500737d7a0c6d625e73e830ff9c1f6" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_init" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bae1df58c5fea7502e8e352ec26b5579f6178e1fdb311e088580c980dee25ed" dependencies = [ "bitflags 1.3.2", - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", - "parking_lot 0.11.2", - "parking_lot_core 0.8.6", + "parking_lot", + "parking_lot_core", "static_init_macro", "winapi", ] [[package]] name = "static_init_macro" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a2595fc3aa78f2d0e45dd425b22282dd863273761cc77780914b2cf3003acf" +checksum = "1389c88ddd739ec6d3f8f83343764a0e944cd23cfbf126a9796a714b0b6edd6f" dependencies = [ - "cfg_aliases", + "cfg_aliases 0.1.1", "memchr", "proc-macro2", "quote", @@ -3472,10 +4170,16 @@ dependencies = [ ] [[package]] -name = "strsim" -version = "0.10.0" +name = "std-semaphore" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structopt" @@ -3503,9 +4207,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -3520,15 +4224,21 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synchronoise" version = "1.0.1" @@ -3540,34 +4250,86 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", - "unicode-xid", + "syn 2.0.114", +] + +[[package]] +name = "syslog-tracing" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d349bc2df408b4bf656709a29643641cef7f1795d708f88b105c626a8f64f6e4" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] name = "tempfile" -version = "3.8.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ - "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.3.5", - "rustix", - "windows-sys", + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.3", + "windows-sys 0.61.2", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -3583,117 +4345,121 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] name = "time" -version = "0.1.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" dependencies = [ "deranged", - "serde", + "num-conv", + "powerfmt", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" dependencies = [ + "num-conv", "time-core", ] [[package]] name = "timeago" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5082dc942361cdfb74eab98bf995762d6015e5bb3a20bf7c5c71213778b4fcb4" +checksum = "a1710e589de0a76aaf295cd47a6699f6405737dbfd3cf2b75c92d000b548d0e6" [[package]] -name = "tinyvec" -version = "1.6.0" +name = "tinystr" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" -version = "1.32.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ - "backtrace", "bytes", "libc", "mio", - "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.6.2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] name = "tokio-io-timeout" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" dependencies = [ "pin-project-lite", "tokio", @@ -3701,24 +4467,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", -] - -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.8", - "tokio", - "webpki", + "syn 2.0.114", ] [[package]] @@ -3727,15 +4482,26 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.6", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -3758,9 +4524,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -3769,14 +4535,13 @@ dependencies = [ "pin-project-lite", "slab", "tokio", - "tracing", ] [[package]] name = "toml" -version = "0.6.0" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -3786,24 +4551,24 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.18.1" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 1.9.3", - "nom8", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", + "winnow", ] [[package]] @@ -3818,10 +4583,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -3863,7 +4628,7 @@ dependencies = [ "rand", "slab", "tokio", - "tokio-util 0.7.8", + "tokio-util 0.7.18", "tower-layer", "tower-service", "tracing", @@ -3871,18 +4636,19 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.5" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "base64 0.13.1", - "bitflags 1.3.2", + "base64 0.21.7", + "bitflags 2.10.0", "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "http-range-header", + "mime", "pin-project-lite", "tower-layer", "tower-service", @@ -3891,23 +4657,22 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3916,20 +4681,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.114", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -3946,26 +4711,37 @@ dependencies = [ ] [[package]] -name = "tracing-log" -version = "0.1.3" +name = "tracing-journald" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -3974,109 +4750,126 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "treediff" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" -dependencies = [ - "serde_json", -] - [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] -name = "unicode-bidi" -version = "0.3.13" +name = "ucd-trie" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] -name = "unicode-xid" -version = "0.2.4" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.0" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] -name = "urlencoding" -version = "2.1.3" +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.2.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ - "getrandom", + "getrandom 0.2.17", + "rand", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vcpkg" @@ -4086,9 +4879,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vsimd" @@ -4098,9 +4891,9 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -4117,58 +4910,51 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.87" +name = "wasm-bindgen" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn 2.0.29", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4176,52 +4962,46 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.29", - "wasm-bindgen-backend", + "syn 2.0.114", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.44", ] [[package]] @@ -4242,11 +5022,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -4256,12 +5036,73 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-targets", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", ] [[package]] @@ -4270,7 +5111,43 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", ] [[package]] @@ -4279,13 +5156,46 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -4294,42 +5204,147 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -4337,62 +5352,166 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] -name = "xmlparser" -version = "0.13.5" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.6" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "yoke" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "linked-hash-map", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", ] [[package]] name = "zeroize" -version = "1.6.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.nix b/Cargo.nix deleted file mode 100644 index 591f60a5..00000000 --- a/Cargo.nix +++ /dev/null @@ -1,6306 +0,0 @@ -# This file was @generated by cargo2nix 0.11.0. -# It is not intended to be manually edited. - -args@{ - release ? true, - rootFeatures ? [ - "garage_db/default" - "garage_util/default" - "garage_rpc/default" - "format_table/default" - "garage_table/default" - "garage_block/default" - "garage_model/default" - "garage_api/default" - "garage_web/default" - "garage/default" - "k2v-client/default" - ], - rustPackages, - buildRustPackages, - hostPlatform, - hostPlatformCpu ? null, - hostPlatformFeatures ? [], - target ? null, - codegenOpts ? null, - profileOpts ? null, - rustcLinkFlags ? null, - rustcBuildFlags ? null, - mkRustCrate, - rustLib, - lib, - workspaceSrc, - ignoreLockHash, -}: -let - nixifiedLockHash = "5df33eefe787762bf831e92c723c153faf8d5910332dcdf2fd941fe03be59936"; - workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; - currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); - lockHashIgnored = if ignoreLockHash - then builtins.trace "Ignoring lock hash" ignoreLockHash - else ignoreLockHash; -in if !lockHashIgnored && (nixifiedLockHash != currentLockHash) then - throw ("Cargo.nix ${nixifiedLockHash} is out of sync with Cargo.lock ${currentLockHash}") -else let - inherit (rustLib) fetchCratesIo fetchCrateLocal fetchCrateGit fetchCrateAlternativeRegistry expandFeatures decideProfile genDrvsByProfile; - profilesByName = { - dev = builtins.fromTOML "lto = \"off\"\n"; - release = builtins.fromTOML "debug = true\n"; - }; - rootFeatures' = expandFeatures rootFeatures; - overridableMkRustCrate = f: - let - drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts rustcLinkFlags rustcBuildFlags; } // (f profileName))); - in { compileMode ? null, profileName ? decideProfile compileMode release }: - let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; }; -in -{ - cargo2nixVersion = "0.11.0"; - workspace = { - garage_db = rustPackages.unknown.garage_db."0.8.4"; - garage_util = rustPackages.unknown.garage_util."0.8.4"; - garage_rpc = rustPackages.unknown.garage_rpc."0.8.4"; - format_table = rustPackages.unknown.format_table."0.1.1"; - garage_table = rustPackages.unknown.garage_table."0.8.4"; - garage_block = rustPackages.unknown.garage_block."0.8.4"; - garage_model = rustPackages.unknown.garage_model."0.8.4"; - garage_api = rustPackages.unknown.garage_api."0.8.4"; - garage_web = rustPackages.unknown.garage_web."0.8.4"; - garage = rustPackages.unknown.garage."0.8.4"; - k2v-client = rustPackages.unknown.k2v-client."0.0.4"; - }; - "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec { - name = "addr2line"; - version = "0.21.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"; }; - dependencies = { - gimli = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gimli."0.28.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "adler"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "ahash"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "getrandom") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "runtime-rng") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "aho-corasick"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "perf-literal" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" = overridableMkRustCrate (profileName: rec { - name = "allocator-api2"; - version = "0.2.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "alloc") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".android-tzdata."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "android-tzdata"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".android_system_properties."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "android_system_properties"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anstream."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "anstream"; - version = "0.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "auto") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "wincon") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstyle" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstyle_parse" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle-parse."0.2.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstyle_query" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle-query."1.0.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "anstyle_wincon" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle-wincon."2.1.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "colorchoice" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".colorchoice."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "utf8parse" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".utf8parse."0.2.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "anstyle"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anstyle-parse."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "anstyle-parse"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "utf8") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "utf8parse" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".utf8parse."0.2.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anstyle-query."1.0.0" = overridableMkRustCrate (profileName: rec { - name = "anstyle-query"; - version = "1.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anstyle-wincon."2.1.0" = overridableMkRustCrate (profileName: rec { - name = "anstyle-wincon"; - version = "2.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstyle" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.75" = overridableMkRustCrate (profileName: rec { - name = "anyhow"; - version = "1.0.75"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" = overridableMkRustCrate (profileName: rec { - name = "arc-swap"; - version = "1.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".arrayvec."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "arrayvec"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".assert-json-diff."2.0.2" = overridableMkRustCrate (profileName: rec { - name = "assert-json-diff"; - version = "2.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"; }; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-compression."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "async-compression"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6"; }; - features = builtins.concatLists [ - [ "libzstd" ] - [ "tokio" ] - [ "zstd" ] - [ "zstd-safe" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - libzstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.4" { inherit profileName; }).out; - zstd_safe = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."6.0.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "async-stream"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_stream_impl" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.5" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "async-stream-impl"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" = overridableMkRustCrate (profileName: rec { - name = "async-trait"; - version = "0.1.73"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "autocfg"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-config."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-config"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9"; }; - features = builtins.concatLists [ - [ "client-hyper" ] - [ "credentials-sso" ] - [ "default" ] - [ "rt-tokio" ] - [ "rustls" ] - ]; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.55.3" { inherit profileName; }).out; - aws_sdk_sso = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-sso."0.28.0" { inherit profileName; }).out; - aws_sdk_sts = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-sts."0.28.0" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" { inherit profileName; }).out; - aws_smithy_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-json."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - fastrand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.9.0" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.28" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zeroize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-credential-types"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae"; }; - dependencies = { - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - fastrand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.9.0" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zeroize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-endpoint"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-http"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44"; }; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.28.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sdk-s3"; - version = "0.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "rt-tokio" ] - [ "rustls" ] - ]; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_endpoint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.55.3" { inherit profileName; }).out; - aws_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.55.3" { inherit profileName; }).out; - aws_sig_auth = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.55.3" { inherit profileName; }).out; - aws_sigv4 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.55.3" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_checksums = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-checksums."0.55.3" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" { inherit profileName; }).out; - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" { inherit profileName; }).out; - aws_smithy_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-json."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_smithy_xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - url = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-sso."0.28.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sdk-sso"; - version = "0.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4"; }; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_endpoint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.55.3" { inherit profileName; }).out; - aws_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.55.3" { inherit profileName; }).out; - aws_sig_auth = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.55.3" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" { inherit profileName; }).out; - aws_smithy_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-json."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-sts."0.28.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sdk-sts"; - version = "0.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b"; }; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_endpoint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.55.3" { inherit profileName; }).out; - aws_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.55.3" { inherit profileName; }).out; - aws_sig_auth = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.55.3" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" { inherit profileName; }).out; - aws_smithy_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-json."0.55.3" { inherit profileName; }).out; - aws_smithy_query = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-query."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - aws_smithy_xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-sig-auth"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "sign-eventstream" ] - ]; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_sigv4 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.55.3" { inherit profileName; }).out; - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-sigv4"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "bytes" ] - [ "default" ] - [ "form_urlencoded" ] - [ "http" ] - [ "percent-encoding" ] - [ "sign-eventstream" ] - [ "sign-http" ] - ]; - dependencies = { - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.28" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-async"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880"; }; - features = builtins.concatLists [ - [ "rt-tokio" ] - ]; - dependencies = { - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-checksums."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-checksums"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - crc32c = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32c."0.6.4" { inherit profileName; }).out; - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - sha1 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha1."0.10.5" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-client"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd"; }; - features = builtins.concatLists [ - [ "client-hyper" ] - [ "rt-tokio" ] - [ "rustls" ] - ]; - dependencies = { - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - fastrand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.9.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - hyper_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-eventstream"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8"; }; - dependencies = { - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-http"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "event-stream" ] - [ "rt-tokio" ] - ]; - dependencies = { - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - bytes_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.3" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - pin_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-http-tower"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-json."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-json"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8"; }; - dependencies = { - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-query."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-query"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d"; }; - dependencies = { - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - urlencoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".urlencoding."2.1.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-types"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8"; }; - dependencies = { - base64_simd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64-simd."0.8.0" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - ryu = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.15" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-xml"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b"; }; - dependencies = { - xmlparser = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.55.3" = overridableMkRustCrate (profileName: rec { - name = "aws-types"; - version = "0.55.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829"; }; - dependencies = { - aws_credential_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-credential-types."0.55.3" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.55.3" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.55.3" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.55.3" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.55.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".backoff."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "backoff"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "instant" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.69" = overridableMkRustCrate (profileName: rec { - name = "backtrace"; - version = "0.3.69"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - ${ if !(hostPlatform.isWindows && hostPlatform.parsed.abi.name == "msvc" && !(hostPlatform.parsed.vendor.name == "uwp")) then "addr2line" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if !(hostPlatform.isWindows && hostPlatform.parsed.abi.name == "msvc" && !(hostPlatform.parsed.vendor.name == "uwp")) then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if !(hostPlatform.isWindows && hostPlatform.parsed.abi.name == "msvc" && !(hostPlatform.parsed.vendor.name == "uwp")) then "miniz_oxide" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.7.1" { inherit profileName; }).out; - ${ if !(hostPlatform.isWindows && hostPlatform.parsed.abi.name == "msvc" && !(hostPlatform.parsed.vendor.name == "uwp")) then "object" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".object."0.32.0" { inherit profileName; }).out; - rustc_demangle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.23" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" = overridableMkRustCrate (profileName: rec { - name = "base64"; - version = "0.13.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" = overridableMkRustCrate (profileName: rec { - name = "base64"; - version = "0.21.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".base64-simd."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "base64-simd"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "detect" ] - [ "std" ] - ]; - dependencies = { - outref = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".outref."0.5.1" { inherit profileName; }).out; - vsimd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".vsimd."0.8.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" = overridableMkRustCrate (profileName: rec { - name = "bincode"; - version = "1.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" = overridableMkRustCrate (profileName: rec { - name = "bitflags"; - version = "1.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.0" = overridableMkRustCrate (profileName: rec { - name = "bitflags"; - version = "2.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" = overridableMkRustCrate (profileName: rec { - name = "blake2"; - version = "0.10.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.4" = overridableMkRustCrate (profileName: rec { - name = "block-buffer"; - version = "0.10.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"; }; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bumpalo."3.13.0" = overridableMkRustCrate (profileName: rec { - name = "bumpalo"; - version = "3.13.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.1" = overridableMkRustCrate (profileName: rec { - name = "bytemuck"; - version = "1.13.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "extern_crate_alloc") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "extern_crate_std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" = overridableMkRustCrate (profileName: rec { - name = "byteorder"; - version = "1.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" = overridableMkRustCrate (profileName: rec { - name = "bytes"; - version = "1.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.3" = overridableMkRustCrate (profileName: rec { - name = "bytes-utils"; - version = "0.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - either = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.9.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" = overridableMkRustCrate (profileName: rec { - name = "bytesize"; - version = "1.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" = overridableMkRustCrate (profileName: rec { - name = "cc"; - version = "1.0.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"; }; - features = builtins.concatLists [ - [ "jobserver" ] - [ "parallel" ] - ]; - dependencies = { - jobserver = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.26" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" = overridableMkRustCrate (profileName: rec { - name = "cfg-if"; - version = "1.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "cfg_aliases"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" = overridableMkRustCrate (profileName: rec { - name = "chrono"; - version = "0.4.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - [ "clock" ] - [ "default" ] - [ "iana-time-zone" ] - [ "js-sys" ] - [ "oldtime" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde") - [ "std" ] - [ "time" ] - [ "wasm-bindgen" ] - [ "wasmbind" ] - [ "winapi" ] - ]; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "android" then "android_tzdata" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".android-tzdata."0.1.1" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "iana_time_zone" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".iana-time-zone."0.1.57" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && !(hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi") then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.1.45" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && !(hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi") then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap."2.34.0" = overridableMkRustCrate (profileName: rec { - name = "clap"; - version = "2.34.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"; }; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - textwrap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".textwrap."0.11.0" { inherit profileName; }).out; - unicode_width = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap."4.4.0" = overridableMkRustCrate (profileName: rec { - name = "clap"; - version = "4.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "color") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "derive") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "env") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "error-context") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "help") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "std") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "suggestions") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "usage") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap_builder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_builder."4.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.4.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap_builder."4.4.0" = overridableMkRustCrate (profileName: rec { - name = "clap_builder"; - version = "4.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "color") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "env") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "error-context") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "help") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "std") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "suggestions") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "usage") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstream" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstream."0.5.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "anstyle" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap_lex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.5.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "strsim" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.4.0" = overridableMkRustCrate (profileName: rec { - name = "clap_derive"; - version = "4.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "heck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.4.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.5.1" = overridableMkRustCrate (profileName: rec { - name = "clap_lex"; - version = "0.5.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".colorchoice."1.0.0" = overridableMkRustCrate (profileName: rec { - name = "colorchoice"; - version = "1.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".core-foundation."0.9.3" = overridableMkRustCrate (profileName: rec { - name = "core-foundation"; - version = "0.9.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"; }; - dependencies = { - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.4" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "core-foundation-sys"; - version = "0.8.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.9" = overridableMkRustCrate (profileName: rec { - name = "cpufeatures"; - version = "0.2.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"; }; - dependencies = { - ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.vendor.name == "apple" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crc32c."0.6.4" = overridableMkRustCrate (profileName: rec { - name = "crc32c"; - version = "0.6.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74"; }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" = overridableMkRustCrate (profileName: rec { - name = "crc32fast"; - version = "1.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.8" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-channel"; - version = "0.5.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"; }; - features = builtins.concatLists [ - [ "crossbeam-utils" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.15" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-epoch"; - version = "0.9.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "alloc") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.16" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "memoffset" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "scopeguard" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.2.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.3.8" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-queue"; - version = "0.3.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "alloc") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.16" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-utils"; - version = "0.8.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "default") - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" = overridableMkRustCrate (profileName: rec { - name = "crypto-common"; - version = "0.1.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.7" { inherit profileName; }).out; - typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling."0.14.4" = overridableMkRustCrate (profileName: rec { - name = "darling"; - version = "0.14.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "suggestions") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_macro" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_macro."0.14.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.4" = overridableMkRustCrate (profileName: rec { - name = "darling_core"; - version = "0.14.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "strsim") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "suggestions") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "fnv" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ident_case" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ident_case."1.0.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "strsim" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling_macro."0.14.4" = overridableMkRustCrate (profileName: rec { - name = "darling_macro"; - version = "0.14.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" = overridableMkRustCrate (profileName: rec { - name = "dashmap"; - version = "4.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - num_cpus = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.16.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".deranged."0.3.8" = overridableMkRustCrate (profileName: rec { - name = "deranged"; - version = "0.3.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".derivative."2.2.0" = overridableMkRustCrate (profileName: rec { - name = "derivative"; - version = "2.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" = overridableMkRustCrate (profileName: rec { - name = "digest"; - version = "0.10.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "block-buffer" ] - [ "core-api" ] - [ "default" ] - [ "mac" ] - [ "std" ] - [ "subtle" ] - ]; - dependencies = { - block_buffer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.4" { inherit profileName; }).out; - crypto_common = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; }).out; - subtle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".subtle."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" = overridableMkRustCrate (profileName: rec { - name = "dirs-next"; - version = "2.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "dirs_sys_next" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-sys-next."0.1.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dirs-sys-next."0.1.2" = overridableMkRustCrate (profileName: rec { - name = "dirs-sys-next"; - version = "0.1.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "redox_users" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_users."0.4.3" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dyn-clone."1.0.13" = overridableMkRustCrate (profileName: rec { - name = "dyn-clone"; - version = "1.0.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".either."1.9.0" = overridableMkRustCrate (profileName: rec { - name = "either"; - version = "1.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"; }; - features = builtins.concatLists [ - [ "default" ] - [ "use_std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.33" = overridableMkRustCrate (profileName: rec { - name = "encoding_rs"; - version = "0.8.33"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".env_logger."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "env_logger"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "auto-color") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "color") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "default") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "humantime") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "regex") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "humantime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".humantime."2.1.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "is_terminal" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".is-terminal."0.4.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "regex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "termcolor" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".equivalent."1.0.1" = overridableMkRustCrate (profileName: rec { - name = "equivalent"; - version = "1.0.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "err-derive"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - proc_macro_error = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - synstructure = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".synstructure."0.12.6" { inherit profileName; }).out; - }; - buildDependencies = { - rustversion = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.14" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".errno."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "errno"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.parsed.kernel.name == "dragonfly" then "errno_dragonfly" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".errno-dragonfly."0.1.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && (hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "hermit" || hostPlatform.parsed.kernel.name == "wasi") then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".errno-dragonfly."0.1.2" = overridableMkRustCrate (profileName: rec { - name = "errno-dragonfly"; - version = "0.1.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "fallible-iterator"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" = overridableMkRustCrate (profileName: rec { - name = "fallible-streaming-iterator"; - version = "0.1.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fastrand."1.9.0" = overridableMkRustCrate (profileName: rec { - name = "fastrand"; - version = "1.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"; }; - dependencies = { - ${ if hostPlatform.parsed.cpu.name == "wasm32" && !(hostPlatform.parsed.kernel.name == "wasi") then "instant" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fastrand."2.0.0" = overridableMkRustCrate (profileName: rec { - name = "fastrand"; - version = "2.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "alloc") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.2" = overridableMkRustCrate (profileName: rec { - name = "fixedbitset"; - version = "0.4.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" = overridableMkRustCrate (profileName: rec { - name = "fnv"; - version = "1.0.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "form_urlencoded"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - }; - }); - - "unknown".format_table."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "format_table"; - version = "0.1.1"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/format-table"); - }); - - "registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "fs2"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "async-await" ] - [ "default" ] - [ "executor" ] - [ "futures-executor" ] - [ "std" ] - ]; - dependencies = { - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.28" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_executor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.28" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.28" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-channel"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "futures-sink" ] - [ "sink" ] - [ "std" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-core"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-executor"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-io"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-macro"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-sink"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-task"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "futures-util"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "async-await" ] - [ "async-await-macro" ] - [ "channel" ] - [ "default" ] - [ "futures-channel" ] - [ "futures-io" ] - [ "futures-macro" ] - [ "futures-sink" ] - [ "io" ] - [ "memchr" ] - [ "sink" ] - [ "slab" ] - [ "std" ] - ]; - dependencies = { - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.28" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.28" { inherit profileName; }).out; - futures_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.28" { profileName = "__noProfile"; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.28" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - pin_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" { inherit profileName; }).out; - slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "fxhash"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - }; - }); - - "unknown".garage."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/garage"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs") - (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery") - (lib.optional (rootFeatures' ? "garage/default") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "opentelemetry-otlp") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus") "opentelemetry-prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus") "prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled") "sled") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite") "sqlite") - (lib.optional (rootFeatures' ? "garage/system-libs") "system-libs") - (lib.optional (rootFeatures' ? "garage/telemetry-otlp") "telemetry-otlp") - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - backtrace = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.69" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; - format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.8.4" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - garage_web = (rustPackages."unknown".garage_web."0.8.4" { inherit profileName; }).out; - git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out; - parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out; - timeago = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.1" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - tracing_subscriber = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.17" { inherit profileName; }).out; - }; - devDependencies = { - assert_json_diff = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".assert-json-diff."2.0.2" { inherit profileName; }).out; - aws_config = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-config."0.55.3" { inherit profileName; }).out; - aws_sdk_s3 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.28.0" { inherit profileName; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out; - }; - }); - - "unknown".garage_api."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_api"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/api"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus") - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - crypto_common = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_range = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-range."0.1.5" { inherit profileName; }).out; - httpdate = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.3" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - idna = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.4.0" { inherit profileName; }).out; - md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" { inherit profileName; }).out; - multer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.1.0" { inherit profileName; }).out; - nom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - quick_xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.26.0" { inherit profileName; }).out; - roxmltree = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.18.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - url = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" { inherit profileName; }).out; - }; - }); - - "unknown".garage_block."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_block"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/block"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "system-libs") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_compression = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-compression."0.4.1" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.4" { inherit profileName; }).out; - }; - }); - - "unknown".garage_db."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_db"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/db"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli") "clap") - (lib.optional (rootFeatures' ? "garage_db/cli") "cli") - (lib.optional (rootFeatures' ? "garage_db/default") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "heed") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "pretty_env_logger") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "rusqlite") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "sled") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "sqlite") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.4.0" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "pretty_env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.5.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.29.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "sled" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - devDependencies = { - mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out; - }; - }); - - "unknown".garage_model."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_model"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/model"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_model/default") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "sled") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "sqlite") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.4" { inherit profileName; }).out; - }; - }); - - "unknown".garage_rpc."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_rpc"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/rpc"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery") "consul-discovery") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive") "err-derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "k8s-openapi") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kubernetes-discovery") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "reqwest") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "schemars") - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "system-libs") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.75.0" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - nix = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nix."0.27.1" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.12" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "unknown".garage_table."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_table"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/table"); - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "unknown".garage_util."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_util"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/util"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - xxhash_rust = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.6" { inherit profileName; }).out; - }; - devDependencies = { - mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "unknown".garage_web."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "garage_web"; - version = "0.8.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/web"); - dependencies = { - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.8.4" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.7" = overridableMkRustCrate (profileName: rec { - name = "generic-array"; - version = "0.14.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"; }; - features = builtins.concatLists [ - [ "more_lengths" ] - ]; - dependencies = { - typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "gethostname"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818"; }; - dependencies = { - ${ if !hostPlatform.isWindows then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" = overridableMkRustCrate (profileName: rec { - name = "getrandom"; - version = "0.2.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".gimli."0.28.0" = overridableMkRustCrate (profileName: rec { - name = "gimli"; - version = "0.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"; }; - features = builtins.concatLists [ - [ "read" ] - [ "read-core" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "git-version"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899"; }; - dependencies = { - git_version_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version-macro."0.3.5" { profileName = "__noProfile"; }).out; - proc_macro_hack = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".git-version-macro."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "git-version-macro"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f"; }; - dependencies = { - proc_macro_hack = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" { profileName = "__noProfile"; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".h2."0.3.21" = overridableMkRustCrate (profileName: rec { - name = "h2"; - version = "0.3.21"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - indexmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.3" { inherit profileName; }).out; - slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.12.3" = overridableMkRustCrate (profileName: rec { - name = "hashbrown"; - version = "0.12.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"; }; - features = builtins.concatLists [ - [ "raw" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.0" = overridableMkRustCrate (profileName: rec { - name = "hashbrown"; - version = "0.14.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "ahash") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "allocator-api2") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "inline-more") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "raw") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "allocator_api2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "hashlink"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f"; }; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" = overridableMkRustCrate (profileName: rec { - name = "heck"; - version = "0.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"; }; - dependencies = { - unicode_segmentation = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-segmentation."1.10.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heck."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "heck"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" = overridableMkRustCrate (profileName: rec { - name = "heed"; - version = "0.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb-rkv-sys") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "bytemuck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed_types" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "lmdb_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "page_size" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "synchronoise" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isWindows then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "heed-traits"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "heed-types"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "bincode") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "serde") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "serde-bincode") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "serde-json") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "serde_json") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "bincode" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "bytemuck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "hermit-abi"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "hex"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "hexdump"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2"; }; - dependencies = { - arrayvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arrayvec."0.5.2" { inherit profileName; }).out; - itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.4.19" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" = overridableMkRustCrate (profileName: rec { - name = "hmac"; - version = "0.12.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"; }; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" = overridableMkRustCrate (profileName: rec { - name = "http"; - version = "0.2.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" = overridableMkRustCrate (profileName: rec { - name = "http-body"; - version = "0.4.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-range."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "http-range"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-range-header."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "http-range-header"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" = overridableMkRustCrate (profileName: rec { - name = "httparse"; - version = "1.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.3" = overridableMkRustCrate (profileName: rec { - name = "httpdate"; - version = "1.0.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".humantime."2.1.0" = overridableMkRustCrate (profileName: rec { - name = "humantime"; - version = "2.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" = overridableMkRustCrate (profileName: rec { - name = "hyper"; - version = "0.14.27"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"; }; - features = builtins.concatLists [ - [ "client" ] - [ "default" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "full") - [ "h2" ] - [ "http1" ] - [ "http2" ] - [ "runtime" ] - [ "server" ] - [ "socket2" ] - [ "stream" ] - [ "tcp" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.28" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - h2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.21" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - httparse = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" { inherit profileName; }).out; - httpdate = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.3" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - socket2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tower_service = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - want = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".want."0.3.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" = overridableMkRustCrate (profileName: rec { - name = "hyper-rustls"; - version = "0.23.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "http1" ] - [ "http2" ] - [ "log" ] - [ "logging" ] - [ "native-tokio" ] - [ "rustls-native-certs" ] - [ "tls12" ] - [ "tokio-runtime" ] - ]; - dependencies = { - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - rustls_native_certs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.6.3" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.23.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.24.1" = overridableMkRustCrate (profileName: rec { - name = "hyper-rustls"; - version = "0.24.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"; }; - features = builtins.concatLists [ - [ "acceptor" ] - [ "default" ] - [ "http1" ] - [ "http2" ] - [ "log" ] - [ "logging" ] - [ "native-tokio" ] - [ "rustls-native-certs" ] - [ "tls12" ] - [ "tokio-runtime" ] - ]; - dependencies = { - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.21.6" { inherit profileName; }).out; - rustls_native_certs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.6.3" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.24.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "hyper-timeout"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_io_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-io-timeout."1.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".iana-time-zone."0.1.57" = overridableMkRustCrate (profileName: rec { - name = "iana-time-zone"; - version = "0.1.57"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"; }; - features = builtins.concatLists [ - [ "fallback" ] - ]; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "android" then "android_system_properties" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".android_system_properties."0.1.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios" then "core_foundation_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.4" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "haiku" then "iana_time_zone_haiku" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".iana-time-zone-haiku."0.1.2" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "windows" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".iana-time-zone-haiku."0.1.2" = overridableMkRustCrate (profileName: rec { - name = "iana-time-zone-haiku"; - version = "0.1.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"; }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ident_case."1.0.1" = overridableMkRustCrate (profileName: rec { - name = "ident_case"; - version = "1.0.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".idna."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "idna"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - unicode_bidi = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-bidi."0.3.13" { inherit profileName; }).out; - unicode_normalization = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-normalization."0.1.22" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.3" = overridableMkRustCrate (profileName: rec { - name = "indexmap"; - version = "1.9.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - hashbrown = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.12.3" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".indexmap."2.0.0" = overridableMkRustCrate (profileName: rec { - name = "indexmap"; - version = "2.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "equivalent" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".equivalent."1.0.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" = overridableMkRustCrate (profileName: rec { - name = "instant"; - version = "0.1.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ipnet."2.8.0" = overridableMkRustCrate (profileName: rec { - name = "ipnet"; - version = "2.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" = overridableMkRustCrate (profileName: rec { - name = "ipnetwork"; - version = "0.20.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".is-terminal."0.4.9" = overridableMkRustCrate (profileName: rec { - name = "is-terminal"; - version = "0.4.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.3.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && !(hostPlatform.isWindows || hostPlatform.parsed.kernel.name == "hermit" || hostPlatform.parsed.kernel.name == "unknown") then "rustix" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."0.38.9" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itertools."0.4.19" = overridableMkRustCrate (profileName: rec { - name = "itertools"; - version = "0.4.19"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "itertools"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"; }; - features = builtins.concatLists [ - [ "default" ] - [ "use_alloc" ] - [ "use_std" ] - ]; - dependencies = { - either = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.9.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" = overridableMkRustCrate (profileName: rec { - name = "itoa"; - version = "1.0.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.26" = overridableMkRustCrate (profileName: rec { - name = "jobserver"; - version = "0.1.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"; }; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" = overridableMkRustCrate (profileName: rec { - name = "js-sys"; - version = "0.3.64"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"; }; - dependencies = { - wasm_bindgen = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" = overridableMkRustCrate (profileName: rec { - name = "json-patch"; - version = "0.2.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "diff") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "treediff") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "treediff" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".jsonpath_lib."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "jsonpath_lib"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - }; - }); - - "unknown".k2v-client."0.0.4" = overridableMkRustCrate (profileName: rec { - name = "k2v-client"; - version = "0.0.4"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/k2v-client"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "clap") - (lib.optional (rootFeatures' ? "k2v-client/cli") "cli") - (lib.optional (rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/format_table") "format_table") - (lib.optional (rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/tracing") "tracing") - (lib.optional (rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/tracing-subscriber") "tracing-subscriber") - ]; - dependencies = { - aws_sigv4 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.55.3" { inherit profileName; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/format_table" then "format_table" else null } = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - hyper_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.24.1" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/tracing" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/tracing-subscriber" then "tracing_subscriber" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.17" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" = overridableMkRustCrate (profileName: rec { - name = "k8s-openapi"; - version = "0.16.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "api") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "http") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "percent-encoding") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "url") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "v1_22") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_value" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde-value."0.7.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9bb19108692aeafebb108fd0a1c381c06ac4c03859652599420975165e939b8a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "config") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-runtime") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "runtime") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-tls") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_client" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-derive."0.75.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_runtime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-runtime."0.75.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-client"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "97e1a80ecd1b1438a2fc004549e155d47250b9e01fbfcf4cfbe9c8b56a085593"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "__non_core") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "base64") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "bytes") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "chrono") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "config") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "dirs") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "either") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "futures") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "http-body") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper-rustls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper-timeout") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpatch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpath_lib") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "pem") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "pin-project") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-pemfile") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-tls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde_yaml") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio-util") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tower") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tower-http") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tracing") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "dirs" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "either" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "jsonpath_lib" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".jsonpath_lib."0.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pem" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pem."1.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rustls_pemfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "secrecy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".secrecy."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_yaml" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_yaml."0.8.26" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-http."0.3.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-core"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f4d780f2bb048eeef64a4c6b2582d26a0fe19e30b4d3cc9e081616e1779c5d47"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "json-patch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpatch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "schema") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "schemars") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "form_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "json_patch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-derive."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-derive"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "98459d53b2841237392cd6959956185b2df15c19d32c3b275ed6ca7b7ee1adae"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling."0.14.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-runtime."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-runtime"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7769af142ee2e46bfa44bd393cf7f40b9d8b80d2e11f6317399551ed17760beb"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "backoff" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backoff."0.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "derivative" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".derivative."2.2.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "json_patch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_client" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "kuska-handshake"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e33da4b69f23c2ece0b3e729d079cebdc2c0206e493e42f510f500ad81c631d5"; }; - features = builtins.concatLists [ - [ "async_std" ] - [ "default" ] - [ "futures" ] - ]; - dependencies = { - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" = overridableMkRustCrate (profileName: rec { - name = "kuska-sodiumoxide"; - version = "0.2.5-0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ae0f8eafdd240b722243787b51fdaf8df6693fb8621d0f7061cdba574214cf88"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - [ "std" ] - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "use-pkg-config") - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - libsodium_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsodium-sys."0.2.7" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" = overridableMkRustCrate (profileName: rec { - name = "lazy_static"; - version = "1.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" = overridableMkRustCrate (profileName: rec { - name = "libc"; - version = "0.2.147"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"; }; - features = builtins.concatLists [ - [ "default" ] - [ "extra_traits" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libsodium-sys."0.2.7" = overridableMkRustCrate (profileName: rec { - name = "libsodium-sys"; - version = "0.2.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6b779387cd56adfbc02ea4a668e704f729be8d6a6abd2c27ca5ee537849a92fd"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "use-pkg-config") - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if !(hostPlatform.parsed.abi.name == "msvc") then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.parsed.abi.name == "msvc" then "libc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { profileName = "__noProfile"; }).out; - pkg_config = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.27" { profileName = "__noProfile"; }).out; - walkdir = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".walkdir."2.3.3" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.26.0" = overridableMkRustCrate (profileName: rec { - name = "libsqlite3-sys"; - version = "0.26.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled_bindings") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "cc") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_14_0") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "pkg-config") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "vcpkg") - ]; - buildDependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.27" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".linked-hash-map."0.5.6" = overridableMkRustCrate (profileName: rec { - name = "linked-hash-map"; - version = "0.5.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.4.5" = overridableMkRustCrate (profileName: rec { - name = "linux-raw-sys"; - version = "0.4.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "errno") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "general") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "ioctl") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "no_std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" = overridableMkRustCrate (profileName: rec { - name = "lmdb-rkv-sys"; - version = "0.11.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.27" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.10" = overridableMkRustCrate (profileName: rec { - name = "lock_api"; - version = "0.4.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"; }; - features = builtins.concatLists [ - [ "atomic_usize" ] - [ "default" ] - ]; - dependencies = { - scopeguard = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.2.0" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" = overridableMkRustCrate (profileName: rec { - name = "log"; - version = "0.4.20"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".matchers."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "matchers"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"; }; - dependencies = { - regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "md-5"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" = overridableMkRustCrate (profileName: rec { - name = "memchr"; - version = "2.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".memoffset."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "memoffset"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "default") - ]; - buildDependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mime."0.3.17" = overridableMkRustCrate (profileName: rec { - name = "mime"; - version = "0.3.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "minimal-lexical"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "miniz_oxide"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"; }; - dependencies = { - adler = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mio."0.8.8" = overridableMkRustCrate (profileName: rec { - name = "mio"; - version = "0.8.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"; }; - features = builtins.concatLists [ - [ "net" ] - [ "os-ext" ] - [ "os-poll" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "mktemp"; - version = "0.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a"; }; - dependencies = { - uuid = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".uuid."1.2.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".multer."2.1.0" = overridableMkRustCrate (profileName: rec { - name = "multer"; - version = "2.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - encoding_rs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.33" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - httparse = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - mime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mime."0.3.17" { inherit profileName; }).out; - spin = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.9.8" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "multimap"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "netapp"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "opentelemetry" ] - [ "opentelemetry-contrib" ] - [ "telemetry" ] - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - kuska_handshake = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - opentelemetry_contrib = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nix."0.27.1" = overridableMkRustCrate (profileName: rec { - name = "nix"; - version = "0.27.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"; }; - features = builtins.concatLists [ - [ "fs" ] - ]; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" = overridableMkRustCrate (profileName: rec { - name = "no-std-net"; - version = "0.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" = overridableMkRustCrate (profileName: rec { - name = "nom"; - version = "7.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - minimal_lexical = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nom8."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "nom8"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nu-ansi-term."0.46.0" = overridableMkRustCrate (profileName: rec { - name = "nu-ansi-term"; - version = "0.46.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"; }; - dependencies = { - overload = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".overload."0.1.1" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "num"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"; }; - features = builtins.concatLists [ - [ "default" ] - [ "num-bigint" ] - [ "std" ] - ]; - dependencies = { - num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; - num_complex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_iter = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" { inherit profileName; }).out; - num_rational = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" = overridableMkRustCrate (profileName: rec { - name = "num-bigint"; - version = "0.2.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "num-complex"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" = overridableMkRustCrate (profileName: rec { - name = "num-integer"; - version = "0.1.45"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" = overridableMkRustCrate (profileName: rec { - name = "num-iter"; - version = "0.1.43"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "num-rational"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"; }; - features = builtins.concatLists [ - [ "bigint" ] - [ "num-bigint" ] - [ "std" ] - ]; - dependencies = { - num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" = overridableMkRustCrate (profileName: rec { - name = "num-traits"; - version = "0.2.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.16.0" = overridableMkRustCrate (profileName: rec { - name = "num_cpus"; - version = "1.16.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"; }; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.3.2" { inherit profileName; }).out; - ${ if !hostPlatform.isWindows then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".object."0.32.0" = overridableMkRustCrate (profileName: rec { - name = "object"; - version = "0.32.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe"; }; - features = builtins.concatLists [ - [ "archive" ] - [ "coff" ] - [ "elf" ] - [ "macho" ] - [ "pe" ] - [ "read_core" ] - [ "unaligned" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" = overridableMkRustCrate (profileName: rec { - name = "once_cell"; - version = "1.18.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "race" ] - [ "std" ] - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "unstable") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".openssl-probe."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "openssl-probe"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry"; - version = "0.17.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"; }; - features = builtins.concatLists [ - [ "async-trait" ] - [ "crossbeam-channel" ] - [ "dashmap" ] - [ "default" ] - [ "fnv" ] - [ "metrics" ] - [ "percent-encoding" ] - [ "pin-project" ] - [ "rand" ] - [ "rt-tokio" ] - [ "tokio" ] - [ "tokio-stream" ] - [ "trace" ] - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - crossbeam_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.8" { inherit profileName; }).out; - dashmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.28" { inherit profileName; }).out; - futures_executor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-contrib"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85637add8f60bb4cac673469c14f47a329c6cec7365c72d72cd32f2d104a721a"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-otlp"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tokio") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tonic") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tonic-build") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_trait" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tonic" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tonic_build" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-prometheus"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "opentelemetry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "protobuf" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ordered-float."2.10.0" = overridableMkRustCrate (profileName: rec { - name = "ordered-float"; - version = "2.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "num_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".outref."0.5.1" = overridableMkRustCrate (profileName: rec { - name = "outref"; - version = "0.5.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".overload."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "overload"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" = overridableMkRustCrate (profileName: rec { - name = "page_size"; - version = "0.4.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" = overridableMkRustCrate (profileName: rec { - name = "parking_lot"; - version = "0.11.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - instant = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - lock_api = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.10" { inherit profileName; }).out; - parking_lot_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" = overridableMkRustCrate (profileName: rec { - name = "parking_lot"; - version = "0.12.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" = overridableMkRustCrate (profileName: rec { - name = "parking_lot_core"; - version = "0.8.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - instant = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - smallvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.8" = overridableMkRustCrate (profileName: rec { - name = "parking_lot_core"; - version = "0.9.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.3.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" = overridableMkRustCrate (profileName: rec { - name = "parse_duration"; - version = "2.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"; }; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - num = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".paste."1.0.14" = overridableMkRustCrate (profileName: rec { - name = "paste"; - version = "1.0.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pem."1.1.1" = overridableMkRustCrate (profileName: rec { - name = "pem"; - version = "1.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" = overridableMkRustCrate (profileName: rec { - name = "percent-encoding"; - version = "2.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.4" = overridableMkRustCrate (profileName: rec { - name = "petgraph"; - version = "0.6.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "fixedbitset" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."2.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" = overridableMkRustCrate (profileName: rec { - name = "pin-project"; - version = "1.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"; }; - dependencies = { - pin_project_internal = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.1.3" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.1.3" = overridableMkRustCrate (profileName: rec { - name = "pin-project-internal"; - version = "1.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" = overridableMkRustCrate (profileName: rec { - name = "pin-project-lite"; - version = "0.2.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "pin-utils"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.27" = overridableMkRustCrate (profileName: rec { - name = "pkg-config"; - version = "0.3.27"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_base"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - no_std_net = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_datalink"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" { inherit profileName; }).out; - pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_sys"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ppv-lite86."0.2.17" = overridableMkRustCrate (profileName: rec { - name = "ppv-lite86"; - version = "0.2.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"; }; - features = builtins.concatLists [ - [ "simd" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "pretty_env_logger"; - version = "0.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".env_logger."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-error"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "syn" ] - [ "syn-error" ] - ]; - dependencies = { - proc_macro_error_attr = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error-attr."1.0.4" { profileName = "__noProfile"; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-error-attr."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-error-attr"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-hack"; - version = "0.5.20+deprecated"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" = overridableMkRustCrate (profileName: rec { - name = "proc-macro2"; - version = "1.0.66"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - ]; - dependencies = { - unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.11" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" = overridableMkRustCrate (profileName: rec { - name = "prometheus"; - version = "0.13.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus") "protobuf") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "fnv" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "memchr" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "protobuf" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-derive") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-build"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "heck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "itertools" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "multimap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "petgraph" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_types" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "regex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tempfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tempfile."3.8.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "which" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".which."4.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-derive"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "anyhow" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.75" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "itertools" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-types"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" = overridableMkRustCrate (profileName: rec { - name = "protobuf"; - version = "2.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.26.0" = overridableMkRustCrate (profileName: rec { - name = "quick-xml"; - version = "0.26.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - [ "serialize" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" = overridableMkRustCrate (profileName: rec { - name = "quote"; - version = "1.0.33"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec { - name = "rand"; - version = "0.8.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "getrandom" ] - [ "libc" ] - [ "rand_chacha" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "small_rng") - [ "std" ] - [ "std_rng" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - rand_chacha = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" { inherit profileName; }).out; - rand_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "rand_chacha"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - ppv_lite86 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ppv-lite86."0.2.17" { inherit profileName; }).out; - rand_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" = overridableMkRustCrate (profileName: rec { - name = "rand_core"; - version = "0.6.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "getrandom" ] - [ "std" ] - ]; - dependencies = { - getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" = overridableMkRustCrate (profileName: rec { - name = "redox_syscall"; - version = "0.2.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"; }; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "redox_syscall"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".redox_users."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "redox_users"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" = overridableMkRustCrate (profileName: rec { - name = "regex"; - version = "1.9.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"; }; - features = builtins.concatLists [ - [ "default" ] - [ "perf" ] - [ "perf-backtrack" ] - [ "perf-cache" ] - [ "perf-dfa" ] - [ "perf-inline" ] - [ "perf-literal" ] - [ "perf-onepass" ] - [ "std" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - ]; - dependencies = { - aho_corasick = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.0.4" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.3.7" { inherit profileName; }).out; - regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.7.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.1.10" = overridableMkRustCrate (profileName: rec { - name = "regex-automata"; - version = "0.1.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"; }; - features = builtins.concatLists [ - [ "default" ] - [ "regex-syntax" ] - [ "std" ] - ]; - dependencies = { - regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.3.7" = overridableMkRustCrate (profileName: rec { - name = "regex-automata"; - version = "0.3.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "dfa-onepass" ] - [ "hybrid" ] - [ "meta" ] - [ "nfa-backtrack" ] - [ "nfa-pikevm" ] - [ "nfa-thompson" ] - [ "perf-inline" ] - [ "perf-literal" ] - [ "perf-literal-multisubstring" ] - [ "perf-literal-substring" ] - [ "std" ] - [ "syntax" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - [ "unicode-word-boundary" ] - ]; - dependencies = { - aho_corasick = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.0.4" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.7.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.29" = overridableMkRustCrate (profileName: rec { - name = "regex-syntax"; - version = "0.6.29"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"; }; - features = builtins.concatLists [ - [ "default" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.7.5" = overridableMkRustCrate (profileName: rec { - name = "regex-syntax"; - version = "0.7.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.20" = overridableMkRustCrate (profileName: rec { - name = "reqwest"; - version = "0.11.20"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "__rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "__tls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "hyper-rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "json") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls-pemfile") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls-tls-manual-roots") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "serde_json") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "tokio-rustls") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "encoding_rs" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "h2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.21" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "hyper_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.24.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "ipnet" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.8.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "mime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mime."0.3.17" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.21.6" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "rustls_pemfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_urlencoded."0.7.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "tokio_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.24.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen_futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-futures."0.4.37" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.64" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.isWindows then "winreg" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winreg."0.50.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" = overridableMkRustCrate (profileName: rec { - name = "ring"; - version = "0.16.20"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "dev_urandom_fallback" ] - [ "once_cell" ] - ]; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.64" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rmp."0.8.12" = overridableMkRustCrate (profileName: rec { - name = "rmp"; - version = "0.8.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.16" { inherit profileName; }).out; - paste = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".paste."1.0.14" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" = overridableMkRustCrate (profileName: rec { - name = "rmp-serde"; - version = "0.15.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d"; }; - dependencies = { - byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - rmp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp."0.8.12" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.18.0" = overridableMkRustCrate (profileName: rec { - name = "roxmltree"; - version = "0.18.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d8f595a457b6b8c6cda66a48503e92ee8d19342f905948f29c383200ec9eb1d8"; }; - features = builtins.concatLists [ - [ "default" ] - [ "positions" ] - [ "std" ] - ]; - dependencies = { - xmlparser = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.29.0" = overridableMkRustCrate (profileName: rec { - name = "rusqlite"; - version = "0.29.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.26.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.23" = overridableMkRustCrate (profileName: rec { - name = "rustc-demangle"; - version = "0.1.23"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "rustc_version"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"; }; - dependencies = { - semver = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".semver."1.0.18" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustix."0.38.9" = overridableMkRustCrate (profileName: rec { - name = "rustix"; - version = "0.38.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "fs") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "std") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "termios") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "use-libc-auxv") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && (!hostPlatform.isWindows && !(hostPlatform.parsed.kernel.name == "linux" && hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64)) || hostPlatform.isWindows) then "libc_errno" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".errno."0.3.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && !hostPlatform.isWindows && !(hostPlatform.parsed.kernel.name == "linux" && hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64)) then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && (hostPlatform.parsed.kernel.name == "linux" && hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64) || (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux") && !(hostPlatform.parsed.kernel.name == "linux" && hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64))) then "linux_raw_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.4.5" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" = overridableMkRustCrate (profileName: rec { - name = "rustls"; - version = "0.20.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "dangerous_configuration") - [ "default" ] - [ "log" ] - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - sct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sct."0.7.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls."0.21.6" = overridableMkRustCrate (profileName: rec { - name = "rustls"; - version = "0.21.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "dangerous_configuration") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - [ "log" ] - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-webpki."0.101.4" { inherit profileName; }).out; - sct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sct."0.7.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.6.3" = overridableMkRustCrate (profileName: rec { - name = "rustls-native-certs"; - version = "0.6.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"; }; - dependencies = { - ${ if hostPlatform.isUnix && !(hostPlatform.parsed.kernel.name == "darwin") then "openssl_probe" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-probe."0.1.5" { inherit profileName; }).out; - rustls_pemfile = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.3" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "schannel" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schannel."0.1.22" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "darwin" then "security_framework" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".security-framework."2.9.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.3" = overridableMkRustCrate (profileName: rec { - name = "rustls-pemfile"; - version = "1.0.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"; }; - dependencies = { - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-webpki."0.101.4" = overridableMkRustCrate (profileName: rec { - name = "rustls-webpki"; - version = "0.101.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.14" = overridableMkRustCrate (profileName: rec { - name = "rustversion"; - version = "1.0.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.15" = overridableMkRustCrate (profileName: rec { - name = "ryu"; - version = "1.0.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" = overridableMkRustCrate (profileName: rec { - name = "same-file"; - version = "1.0.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"; }; - dependencies = { - ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schannel."0.1.22" = overridableMkRustCrate (profileName: rec { - name = "schannel"; - version = "0.1.22"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"; }; - dependencies = { - windows_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.12" = overridableMkRustCrate (profileName: rec { - name = "schemars"; - version = "0.8.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "schemars_derive") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "dyn_clone" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dyn-clone."1.0.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars_derive."0.8.12" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schemars_derive."0.8.12" = overridableMkRustCrate (profileName: rec { - name = "schemars_derive"; - version = "0.8.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde_derive_internals" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive_internals."0.26.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "scopeguard"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sct."0.7.0" = overridableMkRustCrate (profileName: rec { - name = "sct"; - version = "0.7.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"; }; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".secrecy."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "secrecy"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zeroize" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".security-framework."2.9.2" = overridableMkRustCrate (profileName: rec { - name = "security-framework"; - version = "2.9.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"; }; - features = builtins.concatLists [ - [ "OSX_10_9" ] - [ "default" ] - ]; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - core_foundation = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation."0.9.3" { inherit profileName; }).out; - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.4" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - security_framework_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".security-framework-sys."2.9.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".security-framework-sys."2.9.1" = overridableMkRustCrate (profileName: rec { - name = "security-framework-sys"; - version = "2.9.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"; }; - features = builtins.concatLists [ - [ "OSX_10_9" ] - ]; - dependencies = { - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.4" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".semver."1.0.18" = overridableMkRustCrate (profileName: rec { - name = "semver"; - version = "1.0.18"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" = overridableMkRustCrate (profileName: rec { - name = "serde"; - version = "1.0.188"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "derive" ] - [ "rc" ] - [ "serde_derive" ] - [ "std" ] - ]; - dependencies = { - serde_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.188" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde-value."0.7.0" = overridableMkRustCrate (profileName: rec { - name = "serde-value"; - version = "0.7.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ordered_float" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ordered-float."2.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" = overridableMkRustCrate (profileName: rec { - name = "serde_bytes"; - version = "0.11.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.188" = overridableMkRustCrate (profileName: rec { - name = "serde_derive"; - version = "1.0.188"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_derive_internals."0.26.0" = overridableMkRustCrate (profileName: rec { - name = "serde_derive_internals"; - version = "0.26.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" = overridableMkRustCrate (profileName: rec { - name = "serde_json"; - version = "1.0.105"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - [ "default" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "indexmap") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "preserve_order") - [ "std" ] - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."2.0.0" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" { inherit profileName; }).out; - ryu = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.15" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.3" = overridableMkRustCrate (profileName: rec { - name = "serde_spanned"; - version = "0.6.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"; }; - features = builtins.concatLists [ - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_urlencoded."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "serde_urlencoded"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "form_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "itoa" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "ryu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.15" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_yaml."0.8.26" = overridableMkRustCrate (profileName: rec { - name = "serde_yaml"; - version = "0.8.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ryu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.15" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "yaml_rust" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".yaml-rust."0.4.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sha1."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "sha1"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.9" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" = overridableMkRustCrate (profileName: rec { - name = "sha2"; - version = "0.10.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "i686" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.9" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sharded-slab."0.1.4" = overridableMkRustCrate (profileName: rec { - name = "sharded-slab"; - version = "0.1.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"; }; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.1" = overridableMkRustCrate (profileName: rec { - name = "signal-hook-registry"; - version = "1.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" = overridableMkRustCrate (profileName: rec { - name = "slab"; - version = "0.4.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" = overridableMkRustCrate (profileName: rec { - name = "sled"; - version = "0.34.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "no_metrics") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "crc32fast" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "crossbeam_epoch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.15" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.16" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") && (hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows") then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "fxhash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" = overridableMkRustCrate (profileName: rec { - name = "smallvec"; - version = "1.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.9" = overridableMkRustCrate (profileName: rec { - name = "socket2"; - version = "0.4.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"; }; - features = builtins.concatLists [ - [ "all" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".socket2."0.5.3" = overridableMkRustCrate (profileName: rec { - name = "socket2"; - version = "0.5.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"; }; - features = builtins.concatLists [ - [ "all" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "spin"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".spin."0.9.8" = overridableMkRustCrate (profileName: rec { - name = "spin"; - version = "0.9.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"; }; - features = builtins.concatLists [ - [ "mutex" ] - [ "spin_mutex" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" = overridableMkRustCrate (profileName: rec { - name = "static_init"; - version = "1.0.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8a2a1c578e98c1c16fc3b8ec1328f7659a500737d7a0c6d625e73e830ff9c1f6"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out; - ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" { inherit profileName; }).out; - static_init_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - buildDependencies = { - cfg_aliases = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "static_init_macro"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "70a2595fc3aa78f2d0e45dd425b22282dd863273761cc77780914b2cf3003acf"; }; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - buildDependencies = { - cfg_aliases = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "strsim"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" = overridableMkRustCrate (profileName: rec { - name = "structopt"; - version = "0.3.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"; }; - dependencies = { - clap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."2.34.0" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - structopt_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt-derive."0.4.18" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".structopt-derive."0.4.18" = overridableMkRustCrate (profileName: rec { - name = "structopt-derive"; - version = "0.4.18"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"; }; - dependencies = { - heck = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }).out; - proc_macro_error = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".subtle."2.5.0" = overridableMkRustCrate (profileName: rec { - name = "subtle"; - version = "2.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" = overridableMkRustCrate (profileName: rec { - name = "syn"; - version = "1.0.109"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"; }; - features = builtins.concatLists [ - [ "clone-impls" ] - [ "default" ] - [ "derive" ] - [ "extra-traits" ] - [ "full" ] - [ "parsing" ] - [ "printing" ] - [ "proc-macro" ] - [ "quote" ] - [ "visit" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.11" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" = overridableMkRustCrate (profileName: rec { - name = "syn"; - version = "2.0.29"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"; }; - features = builtins.concatLists [ - [ "clone-impls" ] - [ "default" ] - [ "derive" ] - [ "extra-traits" ] - [ "full" ] - [ "parsing" ] - [ "printing" ] - [ "proc-macro" ] - [ "quote" ] - [ "visit" ] - [ "visit-mut" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.11" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.1" = overridableMkRustCrate (profileName: rec { - name = "synchronoise"; - version = "1.0.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "crossbeam_queue" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.3.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".synstructure."0.12.6" = overridableMkRustCrate (profileName: rec { - name = "synstructure"; - version = "0.12.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - unicode_xid = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-xid."0.2.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tempfile."3.8.0" = overridableMkRustCrate (profileName: rec { - name = "tempfile"; - version = "3.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "fastrand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."2.0.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.3.5" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && (hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi") then "rustix" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."0.38.9" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "termcolor"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".textwrap."0.11.0" = overridableMkRustCrate (profileName: rec { - name = "textwrap"; - version = "0.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"; }; - dependencies = { - unicode_width = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.47" = overridableMkRustCrate (profileName: rec { - name = "thiserror"; - version = "1.0.47"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"; }; - dependencies = { - thiserror_impl = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror-impl."1.0.47" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thiserror-impl."1.0.47" = overridableMkRustCrate (profileName: rec { - name = "thiserror-impl"; - version = "1.0.47"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thread_local."1.1.7" = overridableMkRustCrate (profileName: rec { - name = "thread_local"; - version = "1.1.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time."0.1.45" = overridableMkRustCrate (profileName: rec { - name = "time"; - version = "0.1.45"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.10.0+wasi-snapshot-preview1" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time."0.3.28" = overridableMkRustCrate (profileName: rec { - name = "time"; - version = "0.3.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "parsing" ] - [ "std" ] - ]; - dependencies = { - deranged = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".deranged."0.3.8" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - time_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.1" { inherit profileName; }).out; - time_macros = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".time-macros."0.2.14" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "time-core"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time-macros."0.2.14" = overridableMkRustCrate (profileName: rec { - name = "time-macros"; - version = "0.2.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572"; }; - features = builtins.concatLists [ - [ "parsing" ] - ]; - dependencies = { - time_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "timeago"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5082dc942361cdfb74eab98bf995762d6015e5bb3a20bf7c5c71213778b4fcb4"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tinyvec."1.6.0" = overridableMkRustCrate (profileName: rec { - name = "tinyvec"; - version = "1.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "tinyvec_macros" ] - ]; - dependencies = { - tinyvec_macros = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tinyvec_macros."0.1.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tinyvec_macros."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "tinyvec_macros"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" = overridableMkRustCrate (profileName: rec { - name = "tokio"; - version = "1.32.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"; }; - features = builtins.concatLists [ - [ "bytes" ] - [ "default" ] - [ "fs" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "full") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "k2v-client/cli") "io-std") - [ "io-util" ] - [ "libc" ] - [ "macros" ] - [ "mio" ] - [ "net" ] - [ "num_cpus" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "parking_lot") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "process") - [ "rt" ] - [ "rt-multi-thread" ] - [ "signal" ] - [ "signal-hook-registry" ] - [ "socket2" ] - [ "sync" ] - [ "time" ] - [ "tokio-macros" ] - [ "windows-sys" ] - ]; - dependencies = { - ${ if false then "backtrace" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.69" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - mio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.8" { inherit profileName; }).out; - num_cpus = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.1" { inherit profileName; }).out; - socket2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.5.3" { inherit profileName; }).out; - tokio_macros = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-macros."2.1.0" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-io-timeout."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "tokio-io-timeout"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-macros."2.1.0" = overridableMkRustCrate (profileName: rec { - name = "tokio-macros"; - version = "2.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.23.4" = overridableMkRustCrate (profileName: rec { - name = "tokio-rustls"; - version = "0.23.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"; }; - features = builtins.concatLists [ - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.24.1" = overridableMkRustCrate (profileName: rec { - name = "tokio-rustls"; - version = "0.24.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.21.6" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" = overridableMkRustCrate (profileName: rec { - name = "tokio-stream"; - version = "0.1.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"; }; - features = builtins.concatLists [ - [ "default" ] - [ "net" ] - [ "time" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.10" = overridableMkRustCrate (profileName: rec { - name = "tokio-util"; - version = "0.6.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "codec") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_sink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" = overridableMkRustCrate (profileName: rec { - name = "tokio-util"; - version = "0.7.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"; }; - features = builtins.concatLists [ - [ "codec" ] - [ "compat" ] - [ "default" ] - [ "futures-io" ] - [ "io" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "slab") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "time") - [ "tracing" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.28" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.28" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "slab" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" = overridableMkRustCrate (profileName: rec { - name = "toml"; - version = "0.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217"; }; - features = builtins.concatLists [ - [ "default" ] - [ "display" ] - [ "parse" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_spanned = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.3" { inherit profileName; }).out; - toml_datetime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" { inherit profileName; }).out; - toml_edit = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_edit."0.18.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" = overridableMkRustCrate (profileName: rec { - name = "toml_datetime"; - version = "0.5.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5"; }; - features = builtins.concatLists [ - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml_edit."0.18.1" = overridableMkRustCrate (profileName: rec { - name = "toml_edit"; - version = "0.18.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - ]; - dependencies = { - indexmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.3" { inherit profileName; }).out; - nom8 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom8."0.2.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out; - serde_spanned = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.3" { inherit profileName; }).out; - toml_datetime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "tonic"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "async-trait") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "codegen") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "h2") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "hyper") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "hyper-timeout") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-derive") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost1") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tokio") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tower") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tracing-futures") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "transport") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_stream" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_trait" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "h2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.21" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "hyper_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost1" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio_stream" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower_layer" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing_futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "tonic-build"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-build") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "transport") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_build" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.109" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" = overridableMkRustCrate (profileName: rec { - name = "tower"; - version = "0.4.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"; }; - features = builtins.concatLists [ - [ "__common" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "balance") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "buffer") - [ "default" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "discover") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "filter") - [ "futures-core" ] - [ "futures-util" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "indexmap") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "limit") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "load") - [ "log" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "make") - [ "pin-project" ] - [ "pin-project-lite" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "rand") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "ready-cache") - [ "retry" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "slab") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "timeout") - [ "tokio" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio-util") - [ "tracing" ] - [ "util" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.3" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "slab" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.8" { inherit profileName; }).out; - tower_layer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - tower_service = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-http."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "tower-http"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "auth") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "base64") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "map-response-body") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "trace") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tracing") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_range_header" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-range-header."0.3.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_layer" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "tower-layer"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "tower-service"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" = overridableMkRustCrate (profileName: rec { - name = "tracing"; - version = "0.1.37"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"; }; - features = builtins.concatLists [ - [ "attributes" ] - [ "default" ] - [ "log" ] - [ "std" ] - [ "tracing-attributes" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.13" { inherit profileName; }).out; - tracing_attributes = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.26" { profileName = "__noProfile"; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.31" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.26" = overridableMkRustCrate (profileName: rec { - name = "tracing-attributes"; - version = "0.1.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.31" = overridableMkRustCrate (profileName: rec { - name = "tracing-core"; - version = "0.1.31"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "once_cell" ] - [ "std" ] - [ "valuable" ] - ]; - dependencies = { - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - ${ if false then "valuable" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".valuable."0.1.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" = overridableMkRustCrate (profileName: rec { - name = "tracing-futures"; - version = "0.2.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "pin-project") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std-future") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-log."0.1.3" = overridableMkRustCrate (profileName: rec { - name = "tracing-log"; - version = "0.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"; }; - features = builtins.concatLists [ - [ "log-tracer" ] - [ "std" ] - ]; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.31" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.17" = overridableMkRustCrate (profileName: rec { - name = "tracing-subscriber"; - version = "0.3.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "ansi" ] - [ "default" ] - [ "env-filter" ] - [ "fmt" ] - [ "matchers" ] - [ "nu-ansi-term" ] - [ "once_cell" ] - [ "regex" ] - [ "registry" ] - [ "sharded-slab" ] - [ "smallvec" ] - [ "std" ] - [ "thread_local" ] - [ "tracing" ] - [ "tracing-log" ] - ]; - dependencies = { - matchers = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".matchers."0.1.0" { inherit profileName; }).out; - nu_ansi_term = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nu-ansi-term."0.46.0" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.9.4" { inherit profileName; }).out; - sharded_slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sharded-slab."0.1.4" { inherit profileName; }).out; - smallvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.11.0" { inherit profileName; }).out; - thread_local = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thread_local."1.1.7" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.31" { inherit profileName; }).out; - tracing_log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-log."0.1.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" = overridableMkRustCrate (profileName: rec { - name = "treediff"; - version = "3.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde_json") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "with-serde-json") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".try-lock."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "try-lock"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" = overridableMkRustCrate (profileName: rec { - name = "typenum"; - version = "1.16.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-bidi."0.3.13" = overridableMkRustCrate (profileName: rec { - name = "unicode-bidi"; - version = "0.3.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"; }; - features = builtins.concatLists [ - [ "hardcoded-data" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.11" = overridableMkRustCrate (profileName: rec { - name = "unicode-ident"; - version = "1.0.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-normalization."0.1.22" = overridableMkRustCrate (profileName: rec { - name = "unicode-normalization"; - version = "0.1.22"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - tinyvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tinyvec."1.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-segmentation."1.10.1" = overridableMkRustCrate (profileName: rec { - name = "unicode-segmentation"; - version = "1.10.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" = overridableMkRustCrate (profileName: rec { - name = "unicode-width"; - version = "0.1.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-xid."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "unicode-xid"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "untrusted"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".url."2.4.0" = overridableMkRustCrate (profileName: rec { - name = "url"; - version = "2.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; - idna = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.4.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".urlencoding."2.1.3" = overridableMkRustCrate (profileName: rec { - name = "urlencoding"; - version = "2.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".utf8parse."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "utf8parse"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".uuid."1.2.2" = overridableMkRustCrate (profileName: rec { - name = "uuid"; - version = "1.2.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "getrandom" ] - [ "rng" ] - [ "std" ] - [ "v4" ] - ]; - dependencies = { - getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".valuable."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "valuable"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" = overridableMkRustCrate (profileName: rec { - name = "vcpkg"; - version = "0.2.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" = overridableMkRustCrate (profileName: rec { - name = "version_check"; - version = "0.9.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".vsimd."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "vsimd"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "detect" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".walkdir."2.3.3" = overridableMkRustCrate (profileName: rec { - name = "walkdir"; - version = "2.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"; }; - dependencies = { - same_file = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".want."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "want"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"; }; - dependencies = { - try_lock = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".try-lock."0.2.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasi."0.10.0+wasi-snapshot-preview1" = overridableMkRustCrate (profileName: rec { - name = "wasi"; - version = "0.10.0+wasi-snapshot-preview1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" = overridableMkRustCrate (profileName: rec { - name = "wasi"; - version = "0.11.0+wasi-snapshot-preview1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen"; - version = "0.2.87"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"; }; - features = builtins.concatLists [ - [ "default" ] - [ "spans" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - wasm_bindgen_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro."0.2.87" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-backend."0.2.87" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-backend"; - version = "0.2.87"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - bumpalo = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bumpalo."3.13.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - wasm_bindgen_shared = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.87" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-futures."0.4.37" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-futures"; - version = "0.4.37"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && builtins.elem "atomics" hostPlatformFeatures then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.64" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro."0.2.87" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-macro"; - version = "0.2.87"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - wasm_bindgen_macro_support = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro-support."0.2.87" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro-support."0.2.87" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-macro-support"; - version = "0.2.87"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.66" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.33" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.29" { inherit profileName; }).out; - wasm_bindgen_backend = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-backend."0.2.87" { inherit profileName; }).out; - wasm_bindgen_shared = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.87" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.87" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-shared"; - version = "0.2.87"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.64" = overridableMkRustCrate (profileName: rec { - name = "web-sys"; - version = "0.3.64"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "AbortController") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "AbortSignal") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Blob") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "BlobPropertyBag") - [ "Crypto" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Event") - [ "EventTarget" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "File") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "FormData") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Headers") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "MessageEvent") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "ReadableStream") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Request") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestCredentials") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestInit") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestMode") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Response") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "ServiceWorkerGlobalScope") - [ "Window" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Worker") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "WorkerGlobalScope") - ]; - dependencies = { - js_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.64" { inherit profileName; }).out; - wasm_bindgen = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.87" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" = overridableMkRustCrate (profileName: rec { - name = "webpki"; - version = "0.22.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".which."4.4.0" = overridableMkRustCrate (profileName: rec { - name = "which"; - version = "4.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "either" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.isWindows then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.18.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" = overridableMkRustCrate (profileName: rec { - name = "winapi"; - version = "0.3.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"; }; - features = builtins.concatLists [ - [ "consoleapi" ] - [ "errhandlingapi" ] - [ "fileapi" ] - [ "handleapi" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "knownfolders") - [ "minwinbase" ] - [ "minwindef" ] - [ "ntdef" ] - [ "ntsecapi" ] - [ "ntstatus" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "objbase") - [ "processenv" ] - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "processthreadsapi") - [ "profileapi" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "shlobj") - [ "std" ] - [ "synchapi" ] - [ "sysinfoapi" ] - [ "timezoneapi" ] - [ "winbase" ] - [ "wincon" ] - [ "winerror" ] - [ "winnt" ] - [ "winsock2" ] - [ "ws2ipdef" ] - [ "ws2tcpip" ] - [ "wtypesbase" ] - ]; - dependencies = { - ${ if hostPlatform.config == "i686-pc-windows-gnu" then "winapi_i686_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" then "winapi_x86_64_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "winapi-i686-pc-windows-gnu"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "winapi-util"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"; }; - dependencies = { - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "winapi-x86_64-pc-windows-gnu"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows."0.48.0" = overridableMkRustCrate (profileName: rec { - name = "windows"; - version = "0.48.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"; }; - features = builtins.concatLists [ - [ "Globalization" ] - [ "default" ] - ]; - dependencies = { - windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" = overridableMkRustCrate (profileName: rec { - name = "windows-sys"; - version = "0.48.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"; }; - features = builtins.concatLists [ - [ "Win32" ] - [ "Win32_Foundation" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "Win32_NetworkManagement") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "Win32_NetworkManagement_IpHelper") - [ "Win32_Networking" ] - [ "Win32_Networking_WinSock" ] - [ "Win32_Security" ] - [ "Win32_Security_Authentication" ] - [ "Win32_Security_Authentication_Identity" ] - [ "Win32_Security_Credentials" ] - [ "Win32_Security_Cryptography" ] - [ "Win32_Storage" ] - [ "Win32_Storage_FileSystem" ] - [ "Win32_System" ] - [ "Win32_System_Console" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Win32_System_Diagnostics") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Win32_System_Diagnostics_Debug") - [ "Win32_System_IO" ] - [ "Win32_System_Memory" ] - [ "Win32_System_Pipes" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Win32_System_Registry") - [ "Win32_System_SystemServices" ] - [ "Win32_System_Threading" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Win32_System_Time") - [ "Win32_System_WindowsProgramming" ] - [ "default" ] - ]; - dependencies = { - windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows-targets"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"; }; - dependencies = { - ${ if hostPlatform.config == "aarch64-pc-windows-gnullvm" then "windows_aarch64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.abi.name == "msvc" then "windows_aarch64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "gnu" then "windows_i686_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "msvc" then "windows_i686_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.abi.name == "gnu" then "windows_x86_64_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-gnullvm" then "windows_x86_64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.48.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.abi.name == "msvc" then "windows_x86_64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.48.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_aarch64_gnullvm"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_aarch64_msvc"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_i686_gnu"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_i686_msvc"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_gnu"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_gnullvm"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.48.5" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_msvc"; - version = "0.48.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winreg."0.50.0" = overridableMkRustCrate (profileName: rec { - name = "winreg"; - version = "0.50.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.48.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.5" = overridableMkRustCrate (profileName: rec { - name = "xmlparser"; - version = "0.13.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.6" = overridableMkRustCrate (profileName: rec { - name = "xxhash-rust"; - version = "0.8.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"; }; - features = builtins.concatLists [ - [ "xxh3" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".yaml-rust."0.4.5" = overridableMkRustCrate (profileName: rec { - name = "yaml-rust"; - version = "0.4.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "linked_hash_map" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".linked-hash-map."0.5.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zeroize."1.6.0" = overridableMkRustCrate (profileName: rec { - name = "zeroize"; - version = "1.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.4" = overridableMkRustCrate (profileName: rec { - name = "zstd"; - version = "0.12.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - ]; - dependencies = { - zstd_safe = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."6.0.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd-safe."6.0.6" = overridableMkRustCrate (profileName: rec { - name = "zstd-safe"; - version = "6.0.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - [ "std" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - zstd_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-sys."2.0.8+zstd.1.5.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd-sys."2.0.8+zstd.1.5.5" = overridableMkRustCrate (profileName: rec { - name = "zstd-sys"; - version = "2.0.8+zstd.1.5.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - [ "std" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out; - pkg_config = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.27" { profileName = "__noProfile"; }).out; - }; - }); - -} diff --git a/Cargo.toml b/Cargo.toml index b6194766..df4005a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,11 +3,15 @@ resolver = "2" members = [ "src/db", "src/util", + "src/net", "src/rpc", "src/table", "src/block", "src/model", - "src/api", + "src/api/common", + "src/api/s3", + "src/api/k2v", + "src/api/admin", "src/web", "src/garage", "src/k2v-client", @@ -17,19 +21,133 @@ members = [ default-members = ["src/garage"] [workspace.dependencies] + +# Internal Garage crates format_table = { version = "0.1.1", path = "src/format-table" } -garage_api = { version = "0.8.4", path = "src/api" } -garage_block = { version = "0.8.4", path = "src/block" } -garage_db = { version = "0.8.4", path = "src/db", default-features = false } -garage_model = { version = "0.8.4", path = "src/model", default-features = false } -garage_rpc = { version = "0.8.4", path = "src/rpc" } -garage_table = { version = "0.8.4", path = "src/table" } -garage_util = { version = "0.8.4", path = "src/util" } -garage_web = { version = "0.8.4", path = "src/web" } +garage_api_common = { version = "1.3.1", path = "src/api/common" } +garage_api_admin = { version = "1.3.1", path = "src/api/admin" } +garage_api_s3 = { version = "1.3.1", path = "src/api/s3" } +garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" } +garage_block = { version = "1.3.1", path = "src/block" } +garage_db = { version = "1.3.1", path = "src/db", default-features = false } +garage_model = { version = "1.3.1", path = "src/model", default-features = false } +garage_net = { version = "1.3.1", path = "src/net" } +garage_rpc = { version = "1.3.1", path = "src/rpc" } +garage_table = { version = "1.3.1", path = "src/table" } +garage_util = { version = "1.3.1", path = "src/util" } +garage_web = { version = "1.3.1", path = "src/web" } k2v-client = { version = "0.0.4", path = "src/k2v-client" } -[profile.dev] -lto = "off" +# External crates from crates.io +arc-swap = "1.0" +argon2 = "0.5" +async-trait = "0.1.7" +backtrace = "0.3" +base64 = "0.21" +blake2 = "0.10" +bytes = "1.0" +bytesize = "1.1" +cfg-if = "1.0" +chrono = "0.4" +crc32fast = "1.4" +crc32c = "0.6" +crypto-common = "0.1" +gethostname = "0.4" +git-version = "0.3.4" +hex = "0.4" +hexdump = "0.1" +hmac = "0.12" +itertools = "0.12" +ipnet = "2.9.0" +lazy_static = "1.4" +md-5 = "0.10" +mktemp = "0.5" +nix = { version = "0.29", default-features = false, features = ["fs"] } +nom = "7.1" +parking_lot = "0.12" +parse_duration = "2.1" +pin-project = "1.0.12" +pnet_datalink = "0.34" +rand = "0.8" +sha1 = "0.10" +sha2 = "0.10" +timeago = { version = "0.4", default-features = false } +xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } + +aes-gcm = { version = "0.10", features = ["aes", "stream"] } +sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } +kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] } + +clap = { version = "4.1", features = ["derive", "env"] } +pretty_env_logger = "0.5" +structopt = { version = "0.3", default-features = false } +syslog-tracing = "0.3" +tracing = "0.1" +tracing-journald = "0.3.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +heed = { version = "0.11", default-features = false, features = ["lmdb"] } +rusqlite = "0.37" +r2d2 = "0.8" +r2d2_sqlite = "0.31" +fjall = "2.4" + +async-compression = { version = "0.4", features = ["tokio", "zstd"] } +zstd = { version = "0.13", default-features = false } + +quick-xml = { version = "0.26", features = [ "serialize" ] } +rmp-serde = "1.1.2" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_bytes = "0.11" +serde_json = "1.0" +toml = { version = "0.8", default-features = false, features = ["parse"] } + +# newer version requires rust edition 2021 +k8s-openapi = { version = "0.21", features = ["v1_24"] } +kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] } +schemars = "0.8" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] } + +form_urlencoded = "1.0.0" +http = "1.0" +httpdate = "1.0" +http-range = "0.1" +http-body-util = "0.1" +hyper = { version = "1.0", default-features = false } +hyper-util = { version = "0.1", features = [ "full" ] } +multer = "3.0" +percent-encoding = "2.2" +roxmltree = "0.19" +url = "2.3" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +tokio-util = { version = "0.7", features = ["compat", "io"] } +tokio-stream = { version = "0.1", features = ["net"] } + +opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" +opentelemetry-contrib = "0.9" +prometheus = "0.13" + +# used by the k2v-client crate only +aws-sigv4 = { version = "1.1", default-features = false } +hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] } +log = "0.4" +thiserror = "2.0" + +# ---- used only as build / dev dependencies ---- +assert-json-diff = "2.0" +rustc_version = "0.4.0" +static_init = "1.0" +aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] } +aws-sdk-config = { version = "1.62", default-features = false } +aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] } [profile.release] -debug = true +lto = "thin" +codegen-units = 16 +opt-level = 3 +strip = "debuginfo" diff --git a/Dockerfile b/Dockerfile index 62e51b32..2e301ee9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,5 @@ FROM scratch ENV RUST_BACKTRACE=1 ENV RUST_LOG=garage=info -COPY result-bin/bin/garage / +COPY result/bin/garage / CMD [ "/garage", "server"] diff --git a/Makefile b/Makefile index 55fa16dd..35c3f22c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,8 @@ -.PHONY: doc all release shell run1 run2 run3 +.PHONY: doc all run1 run2 run3 all: - clear; cargo build - -release: - nix-build --attr pkgs.amd64.release --no-build-output - -shell: - nix-shell + clear + cargo build # ---- diff --git a/README.md b/README.md index 9992fff2..a0f082a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage) +Garage [![status-badge](https://woodpecker.deuxfleurs.fr/api/badges/1/status.svg)](https://woodpecker.deuxfleurs.fr/repos/1) ===

diff --git a/default.nix b/default.nix index ecdf6436..6974567f 100644 --- a/default.nix +++ b/default.nix @@ -3,54 +3,22 @@ with import ./nix/common.nix; let - pkgs = import pkgsSrc { }; + pkgs = import nixpkgs { }; compile = import ./nix/compile.nix; - build_debug_and_release = (target: { - debug = (compile { - inherit system target git_version pkgsSrc cargo2nixOverlay; - release = false; - }).workspace.garage { compileMode = "build"; }; - - release = (compile { - inherit system target git_version pkgsSrc cargo2nixOverlay; - release = true; - }).workspace.garage { compileMode = "build"; }; - }); - - test = (rustPkgs: - pkgs.symlinkJoin { - name = "garage-tests"; - paths = - builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) - (builtins.attrNames rustPkgs.workspace); - }); + build_release = target: (compile { + inherit target system git_version nixpkgs; + crane = flake.inputs.crane; + rust-overlay = flake.inputs.rust-overlay; + release = true; + }).garage; in { - pkgs = { - amd64 = build_debug_and_release "x86_64-unknown-linux-musl"; - i386 = build_debug_and_release "i686-unknown-linux-musl"; - arm64 = build_debug_and_release "aarch64-unknown-linux-musl"; - arm = build_debug_and_release "armv6l-unknown-linux-musleabihf"; - }; - test = { - amd64 = test (compile { - inherit system git_version pkgsSrc cargo2nixOverlay; - target = "x86_64-unknown-linux-musl"; - features = [ - "garage/bundled-libs" - "garage/k2v" - "garage/sled" - "garage/lmdb" - "garage/sqlite" - ]; - }); - }; - clippy = { - amd64 = (compile { - inherit system git_version pkgsSrc cargo2nixOverlay; - target = "x86_64-unknown-linux-musl"; - compiler = "clippy"; - }).workspace.garage { compileMode = "build"; }; + releasePackages = { + amd64 = build_release "x86_64-unknown-linux-musl"; + i386 = build_release "i686-unknown-linux-musl"; + arm64 = build_release "aarch64-unknown-linux-musl"; + arm = build_release "armv6l-unknown-linux-musleabihf"; }; + flakePackages = flake.packages.${system}; } diff --git a/doc/api/garage-admin-v1.html b/doc/api/garage-admin-v1.html new file mode 100644 index 00000000..783d459e --- /dev/null +++ b/doc/api/garage-admin-v1.html @@ -0,0 +1,24 @@ + + + + Garage Adminstration API v0 + + + + + + + + + + + + + diff --git a/doc/api/garage-admin-v1.yml b/doc/api/garage-admin-v1.yml new file mode 100644 index 00000000..a70dc97b --- /dev/null +++ b/doc/api/garage-admin-v1.yml @@ -0,0 +1,1362 @@ +openapi: 3.0.0 +info: + version: v0.9.0 + title: Garage Administration API v0+garage-v0.9.0 + description: | + Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks. + + *Disclaimer: The API is not stable yet, hence its v0 tag. The API can change at any time, and changes can include breaking backward compatibility. Read the changelog and upgrade your scripts before upgrading. Additionnaly, this specification is very early stage and can contain bugs, especially on error return codes/types that are not tested yet. Do not expect a well finished and polished product!* +paths: + /health: + get: + tags: + - Nodes + operationId: "GetHealth" + summary: "Cluster health report" + description: | + Returns the global status of the cluster, the number of connected nodes (over the number of known ones), the number of healthy storage nodes (over the declared ones), and the number of healthy partitions (over the total). + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Information about the queried node, its environment and the current layout + content: + application/json: + schema: + type: object + required: [ status, knownNodes, connectedNodes, storageNodes, storageNodesOk, partitions, partitionsQuorum, partitionsAllOk ] + properties: + status: + type: string + example: "healthy" + knownNodes: + type: integer + format: int64 + example: 4 + connectedNodes: + type: integer + format: int64 + example: 4 + storageNodes: + type: integer + format: int64 + example: 3 + storageNodesOk: + type: integer + format: int64 + example: 3 + partitions: + type: integer + format: int64 + example: 256 + partitionsQuorum: + type: integer + format: int64 + example: 256 + partitionsAllOk: + type: integer + format: int64 + example: 256 + /status: + get: + tags: + - Nodes + operationId: "GetNodes" + summary: "Describe cluster" + description: | + Returns the cluster's current status, including: + - ID of the node being queried and its version of the Garage daemon + - Live nodes + - Currently configured cluster layout + - Staged changes to the cluster layout + + *Capacity is given in bytes* + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Information about the queried node, its environment and the current layout + content: + application/json: + schema: + type: object + required: [ node, garageVersion, garageFeatures, rustVersion, dbEngine, knownNodes, layout ] + properties: + node: + type: string + example: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + garageVersion: + type: string + example: "v0.9.0" + garageFeatures: + type: array + items: + type: string + example: + - "k2v" + - "lmdb" + - "sqlite" + - "consul-discovery" + - "kubernetes-discovery" + - "metrics" + - "telemetry-otlp" + - "bundled-libs" + rustVersion: + type: string + example: "1.68.0" + dbEngine: + type: string + example: "LMDB (using Heed crate)" + knownNodes: + type: array + example: + - id: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + addr: "10.0.0.11:3901" + isUp: true + lastSeenSecsAgo: 9 + hostname: orion + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + addr: "10.0.0.12:3901" + isUp: true + lastSeenSecsAgo: 13 + hostname: pegasus + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + addr: "10.0.0.13:3901" + isUp: true + lastSeenSecsAgo: 2 + hostname: neptune + items: + $ref: '#/components/schemas/NodeNetworkInfo' + layout: + $ref: '#/components/schemas/ClusterLayout' + + /connect: + post: + tags: + - Nodes + operationId: "AddNode" + summary: "Connect a new node" + description: | + Instructs this Garage node to connect to other Garage nodes at specified `@`. `node_id` is generated automatically on node start. + requestBody: + required: true + content: + application/json: + schema: + type: array + example: + - "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901" + - "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901" + items: + type: string + + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '400': + description: | + Your request is malformed, check your JSON + '200': + description: | + The request has been handled correctly but it does not mean that all connection requests succeeded; some might have fail, you need to check the body! + content: + application/json: + schema: + type: array + example: + - success: true + error: + - success: false + error: "Handshake error" + items: + type: object + properties: + success: + type: boolean + example: true + error: + type: string + nullable: true + example: null + + /layout: + get: + tags: + - Layout + operationId: "GetLayout" + summary: "Details on the current and staged layout" + description: | + Returns the cluster's current layout, including: + - Currently configured cluster layout + - Staged changes to the cluster layout + + *Capacity is given in bytes* + *The info returned by this endpoint is a subset of the info returned by `GET /status`.* + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Returns the cluster's current cluster layout: + - Currently configured cluster layout + - Staged changes to the cluster layout + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterLayout' + + post: + tags: + - Layout + operationId: "AddLayout" + summary: "Send modifications to the cluster layout" + description: | + Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls of `GET /layout`. Once the set of staged changes is satisfactory, the user may call `POST /layout/apply` to apply the changed changes, or `POST /layout/revert` to clear all of the staged changes in the layout. + + Setting the capacity to `null` will configure the node as a gateway. + Otherwise, capacity must be now set in bytes (before Garage 0.9 it was arbitrary weights). + For example to declare 100GB, you must set `capacity: 100000000000`. + + Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes). + requestBody: + description: | + To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`). + To remove a node, simply pass the `remove: true` field. + This logic is represented in OpenAPI with a "One Of" object. + + Contrary to the CLI that may update only a subset of the fields capacity, zone and tags, when calling this API all of these values must be specified. + required: true + content: + application/json: + schema: + type: array + example: + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + zone: "geneva" + capacity: 100000000000 + tags: + - gateway + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + remove: true + items: + $ref: '#/components/schemas/NodeRoleChange' + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The layout modification has been correctly staged" + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterLayout' + + /layout/apply: + post: + tags: + - Layout + operationId: "ApplyLayout" + summary: "Apply staged layout" + description: | + Applies to the cluster the layout changes currently registered as staged layout changes. + + *Note: do not try to parse the `message` field of the response, it is given as an array of string specifically because its format is not stable.* + requestBody: + description: | + Similarly to the CLI, the body must include the version of the new layout that will be created, which MUST be 1 + the value of the currently existing layout in the cluster. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LayoutVersion' + + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The staged layout has been applied as the new layout of the cluster, a rebalance has been triggered." + content: + application/json: + schema: + type: object + required: [ message, layout ] + properties: + message: + type: array + items: + type: string + example: + - "==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====" + - "" + - "Partitions are replicated 1 times on at least 1 distinct zones." + - "" + - "Optimal partition size: 419.4 MB (3 B in previous layout)" + - "Usable capacity / total cluster capacity: 107.4 GB / 107.4 GB (100.0 %)" + - "Effective capacity (replication factor 1): 107.4 GB" + - "" + - "A total of 0 new copies of partitions need to be transferred." + - "" + - "dc1 Tags Partitions Capacity Usable capacity\n 6a8e08af2aab1083 a,v 256 (0 new) 107.4 GB 107.4 GB (100.0%)\n TOTAL 256 (256 unique) 107.4 GB 107.4 GB (100.0%)\n\n" + layout: + $ref: '#/components/schemas/ClusterLayout' + + + /layout/revert: + post: + tags: + - Layout + operationId: "RevertLayout" + summary: "Clear staged layout" + description: | + Clears all of the staged layout changes. + requestBody: + description: | + Reverting the staged changes is done by incrementing the version number and clearing the contents of the staged change list. Similarly to the CLI, the body must include the incremented version number, which MUST be 1 + the value of the currently existing layout in the cluster. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LayoutVersion' + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The staged layout has been cleared, you can start again sending modification from a fresh copy with `POST /layout`." + + "/key?list": + get: + tags: + - Key + operationId: "ListKeys" + summary: "List all keys" + description: | + Returns all API access keys in the cluster. + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string) + content: + application/json: + schema: + type: array + example: + - id: "GK31c2f218a2e44f485b94239e" + name: "test-key" + - id: "GKe10061ac9c2921f09e4c5540" + name: "" + items: + type: object + required: [ id ] + properties: + id: + type: string + name: + type: string + post: + tags: + - Key + operationId: "AddKey" + summary: "Create a new API key" + description: | + Creates a new API access key. + requestBody: + description: | + You can set a friendly name for this key. + If you don't want to, you can set the name to `null`. + + *Note: the secret key is returned in the response.* + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + nullable: true + example: "test-key" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The key has been added" + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + "/key": + get: + tags: + - Key + operationId: "GetKey" + summary: "Get key information" + description: | + Return information about a specific key like its identifiers, its permissions and buckets on which it has permissions. + You can search by specifying the exact key identifier (`id`) or by specifying a pattern (`search`). + + For confidentiality reasons, the secret key is not returned by default: you must pass the `showSecretKey` query parameter to get it. + parameters: + - name: id + in: query + description: | + The exact API access key generated by Garage. + + Incompatible with `search`. + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + - name: search + in: query + description: | + A pattern (beginning or full string) corresponding to a key identifier or friendly name. + + Incompatible with `id`. + example: "test-k" + schema: + type: string + - name: showSecretKey + in: query + schema: + type: string + default: "false" + enum: + - "true" + - "false" + example: "true" + required: false + description: "Wether or not the secret key should be returned in the response" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns information about the key + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + delete: + tags: + - Key + operationId: "DeleteKey" + summary: "Delete a key" + description: | + Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. + parameters: + - name: id + in: query + required: true + description: "The exact API access key generated by Garage" + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: "The key has been deleted" + + + post: + tags: + - Key + operationId: "UpdateKey" + summary: "Update a key" + description: | + Updates information about the specified API access key. + + *Note: the secret key is not returned in the response, `null` is sent instead.* + parameters: + - name: id + in: query + required: true + description: "The exact API access key generated by Garage" + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + requestBody: + description: | + For a given key, provide a first set with the permissions to grant, and a second set with the permissions to remove + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + example: "test-key" + allow: + type: object + example: + properties: + createBucket: + type: boolean + example: true + deny: + type: object + properties: + createBucket: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: | + Returns information about the key + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + + /key/import: + post: + tags: + - Key + operationId: "ImportKey" + summary: "Import an existing key" + description: | + Imports an existing API key. This feature must only be used for migrations and backup restore. + + **Do not use it to generate custom key identifiers or you will break your Garage cluster.** + requestBody: + description: | + Information on the key to import + required: true + content: + application/json: + schema: + type: object + required: [ name, accessKeyId, secretAccessKey ] + properties: + name: + type: string + example: "test-key" + nullable: true + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + secretAccessKey: + type: string + example: "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The key has been imported into the system" + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + "/bucket?list": + get: + tags: + - Bucket + operationId: "ListBuckets" + summary: "List all buckets" + description: | + List all the buckets on the cluster with their UUID and their global and local aliases. + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns the UUID of the bucket and all its aliases + content: + application/json: + schema: + type: array + example: + - id: "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033" + globalAliases: + - "container_registry" + - id: "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95" + localAliases: + - alias: "my_documents" + accessKeyid: "GK31c2f218a2e44f485b94239e" + - id: "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995" + globalAliases: + - "example.com" + - "www.example.com" + localAliases: + - alias: "corp_website" + accessKeyId: "GKe10061ac9c2921f09e4c5540" + - alias: "web" + accessKeyid: "GK31c2f218a2e44f485b94239e" + - id: "" + items: + type: object + required: [ id ] + properties: + id: + type: string + globalAliases: + type: array + items: + type: string + localAliases: + type: array + items: + type: object + required: [ alias, accessKeyId ] + properties: + alias: + type: string + accessKeyId: + type: string + + /bucket: + post: + tags: + - Bucket + operationId: "CreateBucket" + summary: "Create a bucket" + description: | + Creates a new bucket, either with a global alias, a local one, or no alias at all. + Technically, you can also specify both `globalAlias` and `localAlias` and that would create two aliases. + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ ] + properties: + globalAlias: + type: string + example: "my_documents" + localAlias: + type: object + properties: + accessKeyId: + type: string + alias: + type: string + allow: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "The payload is not formatted correctly" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + get: + tags: + - Bucket + operationId: "GetBucketInfo" + summary: "Get a bucket" + description: | + Given a bucket identifier (`id`) or a global alias (`globalAlias`), get its information. + It includes its aliases, its web configuration, keys that have some permissions + on it, some statistics (number of objects, size), number of dangling multipart uploads, + and its quotas (if any). + parameters: + - name: id + in: query + description: | + The exact bucket identifier, a 32 bytes hexadecimal string. + + Incompatible with `alias`. + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + - name: globalAlias + in: query + description: | + The exact global alias of one of the existing buckets. + + Incompatible with `id`. + example: "my_documents" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + + delete: + tags: + - Bucket + operationId: "DeleteBucket" + summary: "Delete a bucket" + description: | + Delete a bucket.Deletes a storage bucket. A bucket cannot be deleted if it is not empty. + + **Warning:** this will delete all aliases associated with the bucket! + parameters: + - name: id + in: query + required: true + description: "The exact bucket identifier, a 32 bytes hexadecimal string" + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bucket is not empty" + '404': + description: "Bucket not found" + '204': + description: Bucket has been deleted + + + + put: + tags: + - Bucket + operationId: "UpdateBucket" + summary: "Update a bucket" + description: | + All fields (`websiteAccess` and `quotas`) are optional. + If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. + + In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. + The field `errorDocument` is optional, if no error document is set a generic + error message is displayed when errors happen. Conversely, if `enabled` is + `false`, neither `indexDocument` nor `errorDocument` must be specified. + + In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null` + to remove the quotas. An absent value will be considered the same as a `null`. It is not possible + to change only one of the two quotas. + parameters: + - name: id + in: query + required: true + description: "The exact bucket identifier, a 32 bytes hexadecimal string" + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + requestBody: + description: | + Requested changes on the bucket. Both root fields are optionals. + required: true + content: + application/json: + schema: + type: object + required: [ ] + properties: + websiteAccess: + type: object + properties: + enabled: + type: boolean + example: true + indexDocument: + type: string + example: "index.html" + errorDocument: + type: string + example: "error/400.html" + quotas: + type: object + properties: + maxSize: + type: integer + format: int64 + nullable: true + example: 19029801 + maxObjects: + type: integer + format: int64 + nullable: true + example: null + + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your body." + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/allow: + post: + tags: + - Bucket + operationId: "AllowBucketKey" + summary: "Allow key" + description: | + ⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious. + + Allows a key to do read/write/owner operations on a bucket. + + Flags in permissions which have the value true will be activated. Other flags will remain unchanged (ie. they will keep their internal value). + + For example, if you set read to true, the key will be allowed to read the bucket. + If you set it to false, the key will keeps its previous read permission. + If you want to disallow read for the key, check the DenyBucketKey operation. + + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ bucketId, accessKeyId, permissions ] + properties: + bucketId: + type: string + example: "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + permissions: + type: object + required: [ read, write, owner ] + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/deny: + post: + tags: + - Bucket + operationId: "DenyBucketKey" + summary: "Deny key" + description: | + ⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious. + + Denies a key from doing read/write/owner operations on a bucket. + + Flags in permissions which have the value true will be deactivated. Other flags will remain unchanged. + + For example, if you set read to true, the key will be denied from reading. + If you set read to false, the key will keep its previous permissions. + If you want the key to have the reading permission, check the AllowBucketKey operation. + + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ bucketId, accessKeyId, permissions ] + properties: + bucketId: + type: string + example: "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + permissions: + type: object + required: [ read, write, owner ] + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/alias/global: + put: + tags: + - Bucket + operationId: "PutBucketGlobalAlias" + summary: "Add a global alias" + description: | + Add a global alias to the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: alias + in: query + required: true + example: my_documents + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + delete: + tags: + - Bucket + operationId: "DeleteBucketGlobalAlias" + summary: "Delete a global alias" + description: | + Delete a global alias from the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: alias + in: query + required: true + schema: + type: string + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/alias/local: + put: + tags: + - Bucket + operationId: "PutBucketLocalAlias" + summary: "Add a local alias" + description: | + Add a local alias, bound to specified account, to the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: accessKeyId + in: query + required: true + schema: + type: string + example: GK31c2f218a2e44f485b94239e + - name: alias + in: query + required: true + schema: + type: string + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + delete: + tags: + - Bucket + operationId: "DeleteBucketLocalAlias" + summary: "Delete a local alias" + description: | + Delete a local alias, bound to specified account, from the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: accessKeyId + in: query + schema: + type: string + required: true + example: GK31c2f218a2e44f485b94239e + - name: alias + in: query + schema: + type: string + required: true + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + schemas: + NodeNetworkInfo: + type: object + required: [ addr, isUp, lastSeenSecsAgo, hostname ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + addr: + type: string + example: "10.0.0.11:3901" + isUp: + type: boolean + example: true + lastSeenSecsAgo: + type: integer + nullable: true + example: 9 + hostname: + type: string + example: "node1" + NodeClusterInfo: + type: object + required: [ id, zone, tags ] + properties: + zone: + type: string + example: dc1 + capacity: + type: integer + format: int64 + nullable: true + example: 4 + tags: + type: array + description: | + User defined tags, put whatever makes sense for you, these tags are not interpreted by Garage + example: + - gateway + - fast + items: + type: string + NodeRoleChange: + oneOf: + - $ref: '#/components/schemas/NodeRoleRemove' + - $ref: '#/components/schemas/NodeRoleUpdate' + NodeRoleRemove: + type: object + required: [ id, remove ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + remove: + type: boolean + example: true + NodeRoleUpdate: + type: object + required: [ id, zone, capacity, tags ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + zone: + type: string + example: "dc1" + capacity: + type: integer + format: int64 + nullable: true + example: 150000000000 + tags: + type: array + items: + type: string + example: + - gateway + - fast + + ClusterLayout: + type: object + required: [ version, roles, stagedRoleChanges ] + properties: + version: + type: integer + example: 12 + roles: + type: array + example: + - id: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + zone: "madrid" + capacity: 300000000000 + tags: + - fast + - amd64 + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + zone: "geneva" + capacity: 700000000000 + tags: + - arm64 + items: + $ref: '#/components/schemas/NodeClusterInfo' + stagedRoleChanges: + type: array + example: + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + zone: "geneva" + capacity: 800000000000 + tags: + - gateway + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + remove: true + items: + $ref: '#/components/schemas/NodeRoleChange' + LayoutVersion: + type: object + required: [ version ] + properties: + version: + type: integer + #format: int64 + example: 13 + + KeyInfo: + type: object + properties: + name: + type: string + example: "test-key" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + secretAccessKey: + type: string + nullable: true + example: "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835" + permissions: + type: object + properties: + createBucket: + type: boolean + example: false + buckets: + type: array + items: + type: object + properties: + id: + type: string + example: "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033" + globalAliases: + type: array + items: + type: string + example: "my-bucket" + localAliases: + type: array + items: + type: string + example: "GK31c2f218a2e44f485b94239e:localname" + permissions: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: false + BucketInfo: + type: object + properties: + id: + type: string + example: afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39 + globalAliases: + type: array + items: + type: string + example: "my_documents" + websiteAccess: + type: boolean + example: true + websiteConfig: + type: object + nullable: true + properties: + indexDocument: + type: string + example: "index.html" + errorDocument: + type: string + example: "error/400.html" + keys: + type: array + items: + $ref: '#/components/schemas/BucketKeyInfo' + objects: + type: integer + format: int64 + example: 14827 + bytes: + type: integer + format: int64 + example: 13189855625 + unfinishedUploads: + type: integer + example: 0 + quotas: + type: object + properties: + maxSize: + nullable: true + type: integer + format: int64 + example: null + maxObjects: + nullable: true + type: integer + format: int64 + example: null + + + BucketKeyInfo: + type: object + properties: + accessKeyId: + type: string + name: + type: string + permissions: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + bucketLocalAliases: + type: array + items: + type: string + example: "my_documents" + + +security: + - bearerAuth: [] + +servers: + - description: A local server + url: http://localhost:3903/v1/ diff --git a/doc/book/build/golang.md b/doc/book/build/golang.md index a508260e..f3f28a40 100644 --- a/doc/book/build/golang.md +++ b/doc/book/build/golang.md @@ -37,30 +37,84 @@ import ( "context" "fmt" "os" + "strings" garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang" ) func main() { - // Set Host and other parameters + // Initialization configuration := garage.NewConfiguration() configuration.Host = "127.0.0.1:3903" - - - // We can now generate a client client := garage.NewAPIClient(configuration) - - // Authentication is handled through the context pattern ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t") - // Send a request - resp, r, err := client.NodesApi.GetNodes(ctx).Execute() - if err != nil { - fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err) - fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + // Nodes + fmt.Println("--- nodes ---") + nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute() + fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname) + capa := int64(1000000000) + change := []garage.NodeRoleChange{ + garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate { + Id: *nodes.KnownNodes[0].Id, + Zone: "dc1", + Capacity: *garage.NewNullableInt64(&capa), + Tags: []string{ "fast", "amd64" }, + }}, } + staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute() + msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute() + fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured - // Process the response - fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname) + health, _, _ := client.NodesApi.GetHealth(ctx).Execute() + fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions) + + // Key + fmt.Println("\n--- key ---") + key := "openapi-key" + keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute() + defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute() + fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get()) + + id := *keyInfo.AccessKeyId + canCreateBucket := true + updateKeyRequest := *garage.NewUpdateKeyRequest() + updateKeyRequest.SetName("openapi-key-updated") + updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket }) + update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute() + fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name) + + keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute() + fmt.Printf("Keys count: %v\n", len(keyList)) + + // Bucket + fmt.Println("\n--- bucket ---") + global_name := "global-ns-openapi-bucket" + local_name := "local-ns-openapi-bucket" + bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{ + GlobalAlias: &global_name, + LocalAlias: &garage.CreateBucketRequestLocalAlias { + AccessKeyId: keyInfo.AccessKeyId, + Alias: &local_name, + }, + }).Execute() + defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute() + fmt.Printf("Bucket id: %s\n", *bucketInfo.Id) + + updateBucketRequest := *garage.NewUpdateBucketRequest() + website := garage.NewUpdateBucketRequestWebsiteAccess() + website.SetEnabled(true) + website.SetIndexDocument("index.html") + website.SetErrorDocument("errors/4xx.html") + updateBucketRequest.SetWebsiteAccess(*website) + quotas := garage.NewUpdateBucketRequestQuotas() + quotas.SetMaxSize(1000000000) + quotas.SetMaxObjects(999999999) + updateBucketRequest.SetQuotas(*quotas) + updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute() + fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess) + + bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute() + fmt.Printf("Bucket count: %v\n", len(bucketList)) } ``` diff --git a/doc/book/build/javascript.md b/doc/book/build/javascript.md index ff009ffe..a065c595 100644 --- a/doc/book/build/javascript.md +++ b/doc/book/build/javascript.md @@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js. A short example: ```javascript -const garage = require('garage_administration_api_v0garage_v0_8_0'); +const garage = require('garage_administration_api_v1garage_v0_9_0'); -const api = new garage.ApiClient("http://127.0.0.1:3903/v0"); +const api = new garage.ApiClient("http://127.0.0.1:3903/v1"); api.authentications['bearerAuth'].accessToken = "s3cr3t"; const [node, layout, key, bucket] = [ diff --git a/doc/book/build/python.md b/doc/book/build/python.md index 5b797897..612394e1 100644 --- a/doc/book/build/python.md +++ b/doc/book/build/python.md @@ -23,7 +23,7 @@ client = minio.Minio( "GKyourapikey", "abcd[...]1234", # Force the region, this is specific to garage - region="region", + region="garage", ) ``` @@ -80,7 +80,7 @@ from garage_admin_sdk.apis import * from garage_admin_sdk.models import * configuration = garage_admin_sdk.Configuration( - host = "http://localhost:3903/v0", + host = "http://localhost:3903/v1", access_token = "s3cr3t" ) @@ -94,13 +94,14 @@ print(f"running garage {status.garage_version}, node_id {status.node}") # Change layout of this node current = layout.get_layout() -layout.add_layout({ - status.node: NodeClusterInfo( +layout.add_layout([ + NodeRoleChange( + id = status.node, zone = "dc1", - capacity = 1, + capacity = 1000000000, tags = [ "dev" ], ) -}) +]) layout.apply_layout(LayoutVersion( version = current.version + 1 )) diff --git a/doc/book/connect/apps/index.md b/doc/book/connect/apps/index.md index 7bad9d09..f52d434b 100644 --- a/doc/book/connect/apps/index.md +++ b/doc/book/connect/apps/index.md @@ -12,7 +12,7 @@ In this section, we cover the following web applications: | [Mastodon](#mastodon) | ✅ | Natively supported | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` | -| [Pixelfed](#pixelfed) | ❓ | Not yet tested | +| [Pixelfed](#pixelfed) | ✅ | Natively supported | | [Pleroma](#pleroma) | ❓ | Not yet tested | | [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Funkwhale](#funkwhale) | ❓ | Not yet tested | @@ -69,7 +69,7 @@ $CONFIG = array( 'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com 'port' => 3900, // Put your reverse proxy port or your S3 API port 'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy - 'region' => 'garage', // Garage has only one region named "garage" + 'region' => 'garage', // Garage default region is named "garage", edit according to your cluster config 'use_path_style' => true // Garage supports only path style, must be set to true ], ], @@ -80,6 +80,53 @@ To test your new configuration, just reload your Nextcloud webpage and start sen *External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html) +#### SSE-C encryption (since Garage v1.0) + +Since version 1.0, Garage supports server-side encryption with customer keys +(SSE-C). In this mode, Garage is responsible for encrypting and decrypting +objects, but it does not store the encryption key itself. The encryption key +should be provided by Nextcloud upon each request. This mode of operation is +supported by Nextcloud and it has successfully been tested together with +Garage. + +To enable SSE-C encryption: + +1. Make sure your Garage server is accessible via SSL through a reverse proxy + such as Nginx, and that it is using a valid public certificate (Nextcloud + might be able to connect to an S3 server that is using a self-signed + certificate, but you will lose many hours while trying, so don't). + Configure values for `use_ssl` and `port` accordingly in your `config.php` + file. + +2. Generate an encryption key using the following command: + + ``` + openssl rand -base64 32 + ``` + + Make sure to keep this key **secret**! + +3. Add the encryption key in your `config.php` file as follows: + + + ```php + [ + 'class' => '\\OC\\Files\\ObjectStore\\S3', + 'arguments' => [ + ... + 'sse_c_key' => 'exampleencryptionkeyLbU+5fKYQcVoqnn+RaIOXgo=', + ... + ], + ], + ``` + +Nextcloud will now make Garage encrypt files at rest in the storage bucket. +These files will not be readable by an S3 client that has credentials to the +bucket but doesn't also know the secret encryption key. + + ### External Storage **From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language). @@ -88,7 +135,7 @@ To test your new configuration, just reload your Nextcloud webpage and start sen *Click on the picture to zoom* Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field. -In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen. +In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen. Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared"). @@ -144,10 +191,10 @@ garage key create peertube-key Keep the Key ID and the Secret key in a pad, they will be needed later. -We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist). +We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists). ```bash -garage bucket create peertube-video -garage bucket create peertube-playlist +garage bucket create peertube-videos +garage bucket create peertube-playlists ``` Now we allow our key to read and write on these buckets: @@ -191,7 +238,7 @@ object_storage: # Put localhost only if you have a garage instance running on that node endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443 - # Garage supports only one region for now, named garage + # Garage default region is named "garage", edit according to your config region: 'garage' credentials: @@ -206,7 +253,7 @@ object_storage: proxify_private_files: false streaming_playlists: - bucket_name: 'peertube-playlist' + bucket_name: 'peertube-playlists' # Keep it empty for our example prefix: '' @@ -216,7 +263,7 @@ object_storage: # Same settings but for webtorrent videos videos: - bucket_name: 'peertube-video' + bucket_name: 'peertube-videos' prefix: '' # You must fill this field to make Peertube use our reverse proxy/website logic base_url: 'http://peertube-videos.web.garage.localhost' @@ -245,7 +292,7 @@ with average object size ranging from 50 KB to 150 KB. As such, your Garage cluster should be configured appropriately for good performance: - use Garage v0.8.0 or higher with the [LMDB database engine](@documentation/reference-manual/configuration.md#db-engine-since-v0-8-0). - With the default Sled database engine, your database could quickly end up taking tens of GB of disk space. + Older versions of Garage used the Sled database engine which had issues, such as databases quickly ending up taking tens of GB of disk space. - the Garage database should be stored on a SSD ### Creating your bucket @@ -288,6 +335,7 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t ```bash $ RAILS_ENV=production bin/tootctl media remove --days 3 +$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles $ RAILS_ENV=production bin/tootctl media remove-orphans $ RAILS_ENV=production bin/tootctl preview_cards remove --days 15 ``` @@ -306,8 +354,6 @@ Imports: 1.7 KB Settings: 0 Bytes ``` -Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567). - ### Migrating your data Data migration should be done with an efficient S3 client. @@ -395,7 +441,7 @@ media_storage_providers: store_synchronous: True # do we want to wait that the file has been written before returning? config: bucket: matrix # the name of our bucket, we chose matrix earlier - region_name: garage # only "garage" is supported for the region field + region_name: garage # "garage" by default, edit according to your cluster config endpoint_url: http://localhost:3900 # the path to the S3 endpoint access_key_id: "GKxxx" # your Key ID secret_access_key: "xxxx" # your Secret Key @@ -421,7 +467,7 @@ Now we can write a simple script (eg `~/.local/bin/matrix-cache-gc`): ## CONFIGURATION ## AWS_ACCESS_KEY_ID=GKxxx AWS_SECRET_ACCESS_KEY=xxxx -S3_ENDPOINT=http://localhost:3900 +AWS_ENDPOINT_URL=http://localhost:3900 S3_BUCKET=matrix MEDIA_STORE=/var/lib/matrix-synapse/media PG_USER=matrix @@ -442,7 +488,7 @@ EOF s3_media_upload update-db 1d s3_media_upload --no-progress check-deleted $MEDIA_STORE -s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $S3_ENDPOINT +s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $AWS_ENDPOINT_URL ``` This script will list all the medias that were not accessed in the 24 hours according to your database. diff --git a/doc/book/connect/backup.md b/doc/book/connect/backup.md index 585ec469..7e97d777 100644 --- a/doc/book/connect/backup.md +++ b/doc/book/connect/backup.md @@ -55,8 +55,8 @@ Create your key and bucket: ```bash garage key create my-key -garage bucket create backup -garage bucket allow backup --read --write --key my-key +garage bucket create backups +garage bucket allow backups --read --write --key my-key ``` Then register your Key ID and Secret key in your environment: @@ -161,3 +161,49 @@ kopia repository validate-provider You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`... Everything should work out-of-the-box. + +## Plakar + +Create your key and bucket on Garage server: + +```bash +garage key create my-plakar-key +garage bucket create plakar-backups +garage bucket allow plakar-backups --read --write --key my-plakar-key +… +``` + +On Plakar server, add your Garage as a storage location: +```bash +plakar store add garageS3 s3://my-garage.tld/plakar-backups \ +region=garage # Or as you've specified in garage.toml \ +access_key= \ +secret_access_key= +``` + +Then create the repository. +```bash +plakar at @garageS3 create -plaintext # Unencrypted +# or +plakar at @garageS3 create #encrypted +``` + +If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups. + + +After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message): +```bash +plakar at @garageS3 check +``` + +Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage: +```bash +$ plakar at ~/backups sync to @garageS3 +``` + +Or list the S3 storage content: +```bash +$ plakar at @garageS3 ls +``` + +More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/ diff --git a/doc/book/connect/cli.md b/doc/book/connect/cli.md index 591ac151..6529e4b2 100644 --- a/doc/book/connect/cli.md +++ b/doc/book/connect/cli.md @@ -70,16 +70,17 @@ Then a file named `~/.aws/config` and put: ```toml [default] region=garage +endpoint_url=http://127.0.0.1:3900 ``` Now, supposing Garage is listening on `http://127.0.0.1:3900`, you can list your buckets with: ```bash -aws --endpoint-url http://127.0.0.1:3900 s3 ls +aws s3 ls ``` -Passing the `--endpoint-url` parameter to each command is annoying but AWS developers do not provide a corresponding configuration entry. -As a workaround, you can redefine the aws command by editing the file `~/.bashrc`: +If you're using awscli `<1.29.0` or `<2.13.0`, you need to pass `--endpoint-url` to each CLI invocation explicitly. +As a workaround, you can redefine the aws command by editing the file `~/.bashrc` in this case: ``` function aws { command aws --endpoint-url http://127.0.0.1:3900 $@ ; } @@ -258,7 +259,7 @@ duck --delete garage:/my-files/an-object.txt ## WinSCP (libs3) {#winscp} -*You can find instructions on how to use the GUI in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).* +*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/prise_en_main/winscp/).* How to use `winscp.com`, the CLI interface of WinSCP: diff --git a/doc/book/connect/repositories.md b/doc/book/connect/repositories.md index 66365d64..537b02e7 100644 --- a/doc/book/connect/repositories.md +++ b/doc/book/connect/repositories.md @@ -17,7 +17,7 @@ Garage can also help you serve this content. ## Gitea -You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachements. +You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachments. You can configure a different target for each data type (check `[lfs]` and `[attachment]` sections of the Gitea documentation) and you can provide a default one through the `[storage]` section. Let's start by creating a key and a bucket (your key id and secret will be needed later, keep them somewhere): diff --git a/doc/book/cookbook/ansible.md b/doc/book/cookbook/ansible.md index 6d624c9c..8b0d2969 100644 --- a/doc/book/cookbook/ansible.md +++ b/doc/book/cookbook/ansible.md @@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below. ## Comparison of Ansible roles -| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | -|------------------------------------|---------------------------------------------|---------------------------------------------------------------| -| **Runtime** | Systemd | Docker | -| **Target OS** | Any Linux | Any Linux | -| **Architecture** | amd64, arm64, i686 | amd64, arm64 | -| **Additional software** | None | Traefik | -| **Automatic node connection** | ❌ | ✅ | -| **Layout management** | ❌ | ✅ | -| **Manage buckets & keys** | ❌ | ✅ (basic) | -| **Allow custom Garage config** | ✅ | ❌ | -| **Facilitate Garage upgrades** | ✅ | ❌ | -| **Multiple instances on one host** | ✅ | ✅ | +| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) | +|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------| +| **Runtime** | Systemd | Docker | Systemd | +| **Target OS** | Any Linux | Any Linux | Any Linux | +| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 | +| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) | +| **Automatic node connection** | ❌ | ✅ | ✅ | +| **Layout management** | ❌ | ✅ | ✅ | +| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ | +| **Allow custom Garage config** | ✅ | ❌ | ❌ | +| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ | +| **Multiple instances on one host** | ✅ | ✅ | ❌ | ## zorun/ansible-role-garage @@ -49,3 +49,15 @@ structured DNS names, etc). As a result, this role makes it easier to start with Garage on Ansible, but is less flexible. + +## eddster2309/ansible-role-garage + +[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/) + +This role is a opinionated but customisable role using the official Garage +static binaries and only requires Systemd. As such it should work on any +Linux based host. It includes all the nesscary configuration to +automatically setup a clustered Garage deployment. Most Garage +configuration options are exposed through Ansible variables so while you +can't provide a custom config you can get very close. It can optionally +installed a HA nginx deployment with Keepalived. diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index 0a6ad8fc..ce6beb7b 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17): apk add garage ``` -The default configuration file is installed to `/etc/garage.toml`. You can run -Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it -will be automatically replaced with a random string on the first start. +The default configuration file is installed to `/etc/garage/garage.toml`. You can run +Garage using: `rc-service garage start`. + +If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start. Please note that this package is built without Consul discovery, Kubernetes discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once @@ -26,7 +27,7 @@ it's stable). ## Arch Linux -Garage is available in the [AUR](https://aur.archlinux.org/packages/garage). +Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage). ## FreeBSD diff --git a/doc/book/cookbook/encryption.md b/doc/book/cookbook/encryption.md index 21a5cbc6..bfbea0ec 100644 --- a/doc/book/cookbook/encryption.md +++ b/doc/book/cookbook/encryption.md @@ -53,20 +53,43 @@ and that's also why your nodes have super long identifiers. Adding TLS support built into Garage is not currently planned. -## Garage stores data in plain text on the filesystem +## Garage stores data in plain text on the filesystem or encrypted using customer keys (SSE-C) -Garage does not handle data encryption at rest by itself, and instead delegates -to the user to add encryption, either at the storage layer (LUKS, etc) or on -the client side (or both). There are no current plans to add data encryption -directly in Garage. +For standard S3 API requests, Garage does not encrypt data at rest by itself. +For the most generic at rest encryption of data, we recommend setting up your +storage partitions on encrypted LUKS devices. -Implementing data encryption directly in Garage might make things simpler for -end users, but also raises many more questions, especially around key -management: for encryption of data, where could Garage get the encryption keys -from ? If we encrypt data but keep the keys in a plaintext file next to them, -it's useless. We probably don't want to have to manage secrets in garage as it -would be very hard to do in a secure way. Maybe integrate with an external -system such as Hashicorp Vault? +If you are developping your own client software that makes use of S3 storage, +we recommend implementing data encryption directly on the client side and never +transmitting plaintext data to Garage. This makes it easy to use an external +untrusted storage provider if necessary. + +Garage does support [SSE-C +encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html), +an encryption mode of Amazon S3 where data is encrypted at rest using +encryption keys given by the client. The encryption keys are passed to the +server in a header in each request, to encrypt or decrypt data at the moment of +reading or writing. The server discards the key as soon as it has finished +using it for the request. This mode allows the data to be encrypted at rest by +Garage itself, but it requires support in the client software. It is also not +adapted to a model where the server is not trusted or assumed to be +compromised, as the server can easily know the encryption keys. Note however +that when using SSE-C encryption, the only Garage node that knows the +encryption key passed in a given request is the node to which the request is +directed (which can be a gateway node), so it is easy to have untrusted nodes +in the cluster as long as S3 API requests containing SSE-C encryption keys are +not directed to them. + +Implementing automatic data encryption directly in Garage without client-side +management of keys (something like +[SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html)) +could make things simpler for end users that don't want to setup LUKS, but also +raises many more questions, especially around key management: for encryption of +data, where could Garage get the encryption keys from? If we encrypt data but +keep the keys in a plaintext file next to them, it's useless. We probably don't +want to have to manage secrets in Garage as it would be very hard to do in a +secure way. At the time of speaking, there are no plans to implement this in +Garage. # Adding data encryption using external tools diff --git a/doc/book/cookbook/exposing-websites.md b/doc/book/cookbook/exposing-websites.md index 5f6a5a28..9382a541 100644 --- a/doc/book/cookbook/exposing-websites.md +++ b/doc/book/cookbook/exposing-websites.md @@ -38,7 +38,7 @@ Our website serving logic is as follow: Now we need to infer the URL of your website through your bucket name. Let assume: - - we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#root_domain)) + - we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#web_root_domain)) - our bucket name is `garagehq.deuxfleurs.fr`. Our bucket will be served if the Host field matches one of these 2 values (the port is ignored): diff --git a/doc/book/cookbook/from-source.md b/doc/book/cookbook/from-source.md index bacf93ab..7105c999 100644 --- a/doc/book/cookbook/from-source.md +++ b/doc/book/cookbook/from-source.md @@ -90,6 +90,6 @@ The following feature flags are available in v0.8.0: | `kubernetes-discovery` | optional | Enable automatic registration and discovery
of cluster nodes through the Kubernetes API | | `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API | | `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry | -| `sled` | *by default* | Enable using Sled to store Garage's metadata | -| `lmdb` | optional | Enable using LMDB to store Garage's metadata | -| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata | +| `syslog` | optional | Enable logging to Syslog | +| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata | +| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata | diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md index dfeb3281..f5bceec8 100644 --- a/doc/book/cookbook/kubernetes.md +++ b/doc/book/cookbook/kubernetes.md @@ -11,7 +11,7 @@ Firstly clone the repository: ```bash git clone https://git.deuxfleurs.fr/Deuxfleurs/garage -cd garage/scripts/helm +cd garage/script/helm ``` Deploy with default options: @@ -26,6 +26,13 @@ Or deploy with custom values: helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml ``` +If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart: + +```bash +kubectl apply -k ../k8s/crd +helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml +``` + After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI: ```bash @@ -86,3 +93,62 @@ helm delete --namespace garage garage ``` Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired. + +## Increase PVC size on running Garage instances + +Since the Garage Helm chart creates the data and meta PVC based on `StatefulSet` templates, increasing the PVC size can be a bit tricky. + +### Confirm the `StorageClass` used for Garage supports volume expansion + +Confirm the storage class used for garage. + +```bash +kubectl -n garage get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE +data-garage-0 Bound pvc-080360c9-8ce3-4acf-8579-1701e57b7f3f 30Gi RWO longhorn-local 77d +data-garage-1 Bound pvc-ab8ba697-6030-4fc7-ab3c-0d6df9e3dbc0 30Gi RWO longhorn-local 5d8h +data-garage-2 Bound pvc-3ab37551-0231-4604-986d-136d0fd950ec 30Gi RWO longhorn-local 5d5h +meta-garage-0 Bound pvc-3b457302-3023-4169-846e-c928c5f2ea65 3Gi RWO longhorn-local 77d +meta-garage-1 Bound pvc-49ace2b9-5c85-42df-9247-51c4cf64b460 3Gi RWO longhorn-local 5d8h +meta-garage-2 Bound pvc-99e2e50f-42b4-4128-ae2f-b52629259723 3Gi RWO longhorn-local 5d5h +``` + +In this case, the storage class is `longhorn-local`. Now, check if `ALLOWVOLUMEEXPANSION` is true for the used `StorageClass`. + +```bash +kubectl get storageclasses.storage.k8s.io longhorn-local +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +longhorn-local driver.longhorn.io Delete Immediate true 103d +``` + +If your `StorageClass` does not support volume expansion, double check if you can enable it. Otherwise, your only real option is to spin up a new Garage cluster with increased size and migrate all data over. + +If your `StorageClass` supports expansion, you are free to continue. + +### Increase the size of the PVCs + +Increase the size of all PVCs to your desired size. + +```bash +kubectl -n garage edit pvc data-garage-0 +kubectl -n garage edit pvc data-garage-1 +kubectl -n garage edit pvc data-garage-2 +kubectl -n garage edit pvc meta-garage-0 +kubectl -n garage edit pvc meta-garage-1 +kubectl -n garage edit pvc meta-garage-2 +``` + +### Increase the size of the `StatefulSet` PVC template + +This is an optional step, but if not done, future instances of Garage will be created with the original size from the template. + +```bash +kubectl -n garage delete sts --cascade=orphan garage +statefulset.apps "garage" deleted +``` + +This will remove the Garage `StatefulSet` but leave the pods running. It may seem destructive but needs to be done this way since edits to the size of PVC templates are prohibited. + +### Redeploy the `StatefulSet` + +Now the size of future PVCs can be increased, and the Garage Helm chart can be upgraded. The new `StatefulSet` should take ownership of the orphaned pods again. diff --git a/doc/book/cookbook/monitoring.md b/doc/book/cookbook/monitoring.md index b204dbbe..d643a9ca 100644 --- a/doc/book/cookbook/monitoring.md +++ b/doc/book/cookbook/monitoring.md @@ -18,7 +18,7 @@ api_bind_addr = "0.0.0.0:3903" ``` This will allow anyone to scrape Prometheus metrics by fetching -`http://localhost:3093/metrics`. If you want to restrict access +`http://localhost:3903/metrics`. If you want to restrict access to the exported metrics, set the `metrics_token` configuration value to a bearer token to be used when fetching the metrics endpoint. diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index a8fbb371..b9927c06 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -19,14 +19,15 @@ To run a real-world deployment, make sure the following conditions are met: - You have at least three machines with sufficient storage space available. -- Each machine has a public IP address which is reachable by other machines. It - is highly recommended that you use IPv6 for this end-to-end connectivity. If - IPv6 is not available, then using a mesh VPN such as +- Each machine has an IP address which makes it directly reachable by all other machines. + In many cases, nodes will be behind a NAT and will not each have a public + IPv4 addresses. In this case, is recommended that you use IPv6 for this + end-to-end connectivity if it is available. Otherwise, using a mesh VPN such as [Nebula](https://github.com/slackhq/nebula) or [Yggdrasil](https://yggdrasil-network.github.io/) are approaches to consider in addition to building out your own VPN tunneling. -- This guide will assume you are using Docker containers to deploy Garage on each node. +- This guide will assume you are using Docker containers to deploy Garage on each node. Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md). You can also use an orchestrator such as Nomad or Kubernetes to automatically manage Docker containers on a fleet of nodes. @@ -42,7 +43,7 @@ For our example, we will suppose the following infrastructure with IPv6 connecti | Brussels | Mars | fc00:F::1 | 1.5 TB | Note that Garage will **always** store the three copies of your data on nodes at different -locations. This means that in the case of this small example, the available capacity +locations. This means that in the case of this small example, the usable capacity of the cluster is in fact only 1.5 TB, because nodes in Brussels can't store more than that. This also means that nodes in Paris and London will be under-utilized. To make better use of the available hardware, you should ensure that the capacity @@ -52,9 +53,9 @@ to store 2 TB of data in total. ### Best practices -- If you have fast dedicated networking between all your nodes, and are planing to store - very large files, bump the `block_size` configuration parameter to 10 MB - (`block_size = 10485760`). +- If you have reasonably fast networking between all your nodes, and are planing to store + mostly large files, bump the `block_size` configuration parameter to 10 MB + (`block_size = "10M"`). - Garage stores its files in two locations: it uses a metadata directory to store frequently-accessed small metadata items, and a data directory to store data blocks of uploaded objects. @@ -67,31 +68,42 @@ to store 2 TB of data in total. EXT4 is not recommended as it has more strict limitations on the number of inodes, which might cause issues with Garage when large numbers of objects are stored. -- If you only have an HDD and no SSD, it's fine to put your metadata alongside the data - on the same drive. Having lots of RAM for your kernel to cache the metadata will - help a lot with performance. Make sure to use the LMDB database engine, - instead of Sled, which suffers from quite bad performance degradation on HDDs. - Sled is still the default for legacy reasons, but is not recommended anymore. - -- For the metadata storage, Garage does not do checksumming and integrity - verification on its own. If you are afraid of bitrot/data corruption, - put your metadata directory on a ZFS or BTRFS partition. Otherwise, just use regular - EXT4 or XFS. - - Servers with multiple HDDs are supported natively by Garage without resorting to RAID, see [our dedicated documentation page](@/documentation/operations/multi-hdd.md). +- For the metadata storage, Garage does not do checksumming and integrity + verification on its own, so it is better to use a robust filesystem such as + BTRFS or ZFS. Users have reported that when using the LMDB database engine + (the default), database files have a tendency of becoming corrupted after an + unclean shutdown (e.g. a power outage), so you should take regular snapshots + to be able to recover from such a situation. This can be done using Garage's + built-in automatic snapshotting (since v0.9.4), or by using filesystem level + snapshots. If you cannot do so, you might want to switch to Sqlite which is + more robust. + +- LMDB is the fastest and most tested database engine, but it has the following + weaknesses: 1/ data files are not architecture-independent, you cannot simply + move a Garage metadata directory between nodes running different architectures, + and 2/ LMDB is not suited for 32-bit platforms. Sqlite is a viable alternative + if any of these are of concern. + +- If you only have an HDD and no SSD, it's fine to put your metadata alongside + the data on the same drive, but then consider your filesystem choice wisely + (see above). Having lots of RAM for your kernel to cache the metadata will + help a lot with performance. The default LMDB database engine is the most + tested and has good performance. + ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v0.8.0`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v0.8.0` but it's up to you +We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v0.8.0 +sudo docker pull dxflrs/garage:v1.3.0 ``` ## Deploying and configuring Garage @@ -114,8 +126,9 @@ A valid `/etc/garage.toml` for our cluster would look as follows: metadata_dir = "/var/lib/garage/meta" data_dir = "/var/lib/garage/data" db_engine = "lmdb" +metadata_auto_snapshot_interval = "6h" -replication_mode = "3" +replication_factor = 3 compression_level = 2 @@ -139,6 +152,8 @@ Check the following for your configuration files: - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. This parameter is optional but recommended: if your nodes have trouble communicating with one another, consider adding it. + Alternatively, you can also set `rpc_public_addr_subnet`, which can filter + the addresses announced to other peers to a specific subnet. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. You can generate such a key with `openssl rand -hex 32`. @@ -156,12 +171,13 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v0.8.0 + dxflrs/garage:v1.3.0 ``` -It should be restarted automatically at each reboot. -Please note that we use host networking as otherwise Docker containers -can not communicate with IPv6. +With this command line, Garage should be started automatically at each boot. +Please note that we use host networking as otherwise the network indirection +added by Docker would prevent Garage nodes from communicating with one another +(especially if using IPv6). If you want to use `docker-compose`, you may use the following `docker-compose.yml` file as a reference: @@ -169,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v0.8.0 + image: dxflrs/garage:v1.3.0 network_mode: "host" restart: unless-stopped volumes: @@ -178,12 +194,14 @@ services: - /var/lib/garage/data:/var/lib/garage/data ``` -Upgrading between Garage versions should be supported transparently, -but please check the relase notes before doing so! -To upgrade, simply stop and remove this container and -start again the command with a new version of Garage. +If you wish to upgrade your cluster, make sure to read the corresponding +[documentation page](@/documentation/operations/upgrading.md) first, as well as +the documentation relevant to your version of Garage in the case of major +upgrades. With the containerized setup proposed here, the upgrade process +will require stopping and removing the existing container, and re-creating it +with the upgraded version. -## Controling the daemon +## Controlling the daemon The `garage` binary has two purposes: - it acts as a daemon when launched with `garage server` @@ -241,7 +259,7 @@ You can then instruct nodes to connect to one another as follows: Venus$ garage node connect 563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901 ``` -You don't nead to instruct all node to connect to all other nodes: +You don't need to instruct all node to connect to all other nodes: nodes will discover one another transitively. Now if your run `garage status` on any node, you should have an output that looks as follows: @@ -265,12 +283,12 @@ of a role that is assigned to each active cluster node. For our example, we will suppose we have the following infrastructure (Capacity, Identifier and Zone are specific values to Garage described in the following): -| Location | Name | Disk Space | `Capacity` | `Identifier` | `Zone` | -|----------|---------|------------|------------|--------------|--------------| -| Paris | Mercury | 1 TB | `10` | `563e` | `par1` | -| Paris | Venus | 2 TB | `20` | `86f0` | `par1` | -| London | Earth | 2 TB | `20` | `6814` | `lon1` | -| Brussels | Mars | 1.5 TB | `15` | `212f` | `bru1` | +| Location | Name | Disk Space | Identifier | Zone (`-z`) | Capacity (`-c`) | +|----------|---------|------------|------------|-------------|-----------------| +| Paris | Mercury | 1 TB | `563e` | `par1` | `1T` | +| Paris | Venus | 2 TB | `86f0` | `par1` | `2T` | +| London | Earth | 2 TB | `6814` | `lon1` | `2T` | +| Brussels | Mars | 1.5 TB | `212f` | `bru1` | `1.5T` | #### Node identifiers @@ -292,6 +310,8 @@ garage status It will display the IP address associated with each node; from the IP address you will be able to recognize the node. +We will now use the `garage layout assign` command to configure the correct parameters for each node. + #### Zones Zones are simply a user-chosen identifier that identify a group of server that are grouped together logically. @@ -301,29 +321,29 @@ In most cases, a zone will correspond to a geographical location (i.e. a datacen Behind the scene, Garage will use zone definition to try to store the same data on different zones, in order to provide high availability despite failure of a zone. +Zones are passed to Garage using the `-z` flag of `garage layout assign` (see below). + #### Capacity -Garage reasons on an abstract metric about disk storage that is named the *capacity* of a node. -The capacity configured in Garage must be proportional to the disk space dedicated to the node. +Garage needs to know the storage capacity (disk space) it can/should use on +each node, to be able to correctly balance data. -Capacity values must be **integers** but can be given any signification. -Here we chose that 1 unit of capacity = 100 GB. +Capacity values are expressed in bytes and are passed to Garage using the `-c` flag of `garage layout assign` (see below). -Note that the amount of data stored by Garage on each server may not be strictly proportional to -its capacity value, as Garage will priorize having 3 copies of data in different zones, -even if this means that capacities will not be strictly respected. For example in our above examples, -nodes Earth and Mars will always store a copy of everything each, and the third copy will -have 66% chance of being stored by Venus and 33% chance of being stored by Mercury. +#### Tags + +You can add additional tags to nodes using the `-t` flag of `garage layout assign` (see below). +Tags have no specific meaning for Garage and can be used at your convenience. #### Injecting the topology Given the information above, we will configure our cluster as follow: ```bash -garage layout assign 563e -z par1 -c 10 -t mercury -garage layout assign 86f0 -z par1 -c 20 -t venus -garage layout assign 6814 -z lon1 -c 20 -t earth -garage layout assign 212f -z bru1 -c 15 -t mars +garage layout assign 563e -z par1 -c 1T -t mercury +garage layout assign 86f0 -z par1 -c 2T -t venus +garage layout assign 6814 -z lon1 -c 2T -t earth +garage layout assign 212f -z bru1 -c 1.5T -t mars ``` At this point, the changes in the cluster layout have not yet been applied. @@ -333,6 +353,7 @@ To show the new layout that will be applied, call: garage layout show ``` +Make sure to read carefully the output of `garage layout show`. Once you are satisfied with your new layout, apply it with: ```bash diff --git a/doc/book/cookbook/reverse-proxy.md b/doc/book/cookbook/reverse-proxy.md index b715193e..bdc1c549 100644 --- a/doc/book/cookbook/reverse-proxy.md +++ b/doc/book/cookbook/reverse-proxy.md @@ -472,3 +472,32 @@ https:// { More information on how this endpoint is implemented in Garage is available in the [Admin API Reference](@/documentation/reference-manual/admin-api.md) page. + +### Fileserver browser + +Caddy's built-in +[file_server](https://caddyserver.com/docs/caddyfile/directives/file_server) +browser functionality can be extended with the +[caddy-fs-s3](https://github.com/sagikazarmark/caddy-fs-s3) module. + +This can be configured to use Garage as a backend with the following +configuration: + +```caddy +browse.garage.tld { + file_server { + fs s3 { + bucket test-bucket + region garage + + endpoint https://s3.garage.tld + use_path_style + } + + browse + } +} +``` + +Caddy must also be configured with the required `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` environment variables to access the bucket. diff --git a/doc/book/cookbook/systemd.md b/doc/book/cookbook/systemd.md index c0ed7d1f..ebff8c15 100644 --- a/doc/book/cookbook/systemd.md +++ b/doc/book/cookbook/systemd.md @@ -28,6 +28,7 @@ StateDirectory=garage DynamicUser=true ProtectHome=true NoNewPrivileges=true +LimitNOFILE=42000 [Install] WantedBy=multi-user.target diff --git a/doc/book/design/goals.md b/doc/book/design/goals.md index 78ac7978..efa3cd33 100644 --- a/doc/book/design/goals.md +++ b/doc/book/design/goals.md @@ -48,7 +48,22 @@ locations. They use Garage themselves for the following tasks: - As a backup target using `rclone` and `restic` -- In the Drone continuous integration platform to store task logs - The Deuxfleurs Garage cluster is a multi-site cluster currently composed of 9 nodes in 3 physical locations. + +### Triplebit + +[Triplebit](https://www.triplebit.org) is a non-profit hosting provider and +ISP focused on improving access to privacy-related services. They use +Garage themselves for the following tasks: + +- Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites + +- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/) + +- As a PeerTube storage backend for [neat.tube](https://neat.tube/) + +- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider) + +Triplebit's Garage cluster is a multi-site cluster currently composed of +10 nodes in 3 physical locations. diff --git a/doc/book/design/internals.md b/doc/book/design/internals.md index cefb7acc..8e3c214e 100644 --- a/doc/book/design/internals.md +++ b/doc/book/design/internals.md @@ -97,7 +97,7 @@ delete a tombstone, the following condition has to be met: superseeded by the tombstone. This ensures that deleting the tombstone is safe and that no deleted value will come back in the system. -Garage makes use of Sled's atomic operations (such as compare-and-swap and +Garage uses atomic database operations (such as compare-and-swap and transactions) to ensure that only tombstones that have been correctly propagated to other nodes are ever deleted from the local entry tree. diff --git a/doc/book/design/related-work.md b/doc/book/design/related-work.md index 6c1a6b12..84e66c4e 100644 --- a/doc/book/design/related-work.md +++ b/doc/book/design/related-work.md @@ -67,7 +67,7 @@ Pithos has been abandonned and should probably not used yet, in the following we Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too). From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment. They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it. -We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. +We considered their v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. **[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):** *Not written yet* diff --git a/doc/book/development/devenv.md b/doc/book/development/devenv.md index dd3bdec0..518fd232 100644 --- a/doc/book/development/devenv.md +++ b/doc/book/development/devenv.md @@ -36,7 +36,7 @@ sudo killall nix-daemon Now you can enter our nix-shell, all the required packages will be downloaded but they will not pollute your environment outside of the shell: ```bash -nix-shell +nix-shell -A devShell ``` You can use the traditional Rust development workflow: @@ -65,8 +65,8 @@ nix-build -j $(nproc) --max-jobs auto ``` Our build has multiple parameters you might want to set: - - `release` build with release optimisations instead of debug - - `target allows` for cross compilation + - `release` to build with release optimisations instead of debug + - `target` allows for cross compilation - `compileMode` can be set to test or bench to build a unit test runner - `git_version` to inject the hash to display when running `garage stats` @@ -80,7 +80,7 @@ nix-build \ --git_version $(git rev-parse HEAD) ``` -*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.* +*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.* If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`: diff --git a/doc/book/development/miscellaneous-notes.md b/doc/book/development/miscellaneous-notes.md index f0083ae5..a421943f 100644 --- a/doc/book/development/miscellaneous-notes.md +++ b/doc/book/development/miscellaneous-notes.md @@ -81,12 +81,9 @@ Our cache will be checked. - http://www.lpenz.org/articles/nixchannel/index.html -## Drone +## Woodpecker -Do not try to set a build as trusted from the interface or the CLI tool, -your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`). - -Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. +Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. ## Building Docker containers @@ -99,3 +96,4 @@ We were: - Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json. Finally we chose to build kaniko through nix and use it in a `nix-shell`. +We then switched to using kaniko from nixpkgs when it was packaged. diff --git a/doc/book/development/release-process.md b/doc/book/development/release-process.md index 3fed4add..0c6701c0 100644 --- a/doc/book/development/release-process.md +++ b/doc/book/development/release-process.md @@ -42,7 +42,7 @@ and the docker containers on Docker Hub. ## Automation -We automated our release process with Nix and Drone to make it more reliable. +We automated our release process with Nix and Woodpecker to make it more reliable. Here we describe how we have done in case you want to debug or improve it. ### Caching build steps @@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example: nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' result ``` -*Note that you need the signing key. In our case, it is stored as a secret in Drone.* +*The signing key possessed by the Garage maintainers is required to update the Nix cache.* -The previous command will only send the built packet and not its dependencies. -To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol. - -Instead, you can use the following commands to list all the runtime dependencies: +The previous command will only send the built package and not its dependencies. +In the case of our CI pipeline, we want to cache all intermediate build steps +as well. This can be done using this quite involved command (here as an example +for the `pkgs.amd64.relase` package): ```bash -nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR result/) +nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/nix-signing-key.sec' \ + $(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') ``` -*We could also write this expression with xargs but this tool is not available in our container.* +This command will simultaneously build all of the required Nix paths (using at +most 8 parallel Nix builder jobs) and send the resulting objects to the cache. -But in certain cases, we want to cache compile time dependencies also. -For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own. -We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it. - -This time, the command is a bit more involved: - -```bash -nix copy --to \ - 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate)) -``` - -This is the command we use in our CI as we expect the final binary to change, so we mainly focus on -caching our development dependencies. - -*Currently there is no automatic garbage collection of the cache: we should monitor its growth. -Hopefully, we can erase it totally without breaking any build, the next build will only be slower.* - -In practise, we concluded that we do not want to cache all the compilation dependencies. -Instead, we want to cache the toolchain we use to build Garage each time we change it. -So we removed from Drone any automatic update of the cache and instead handle them manually with: +This can be run for all the Garage packages we build using the following command: ``` source ~/.awsrc -nix-shell --run 'refresh_toolchain' +nix-shell --attr cache --run 'refresh_cache' ``` -Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache. - -To erase the cache: +We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.* +This means we should also monitor the cache's size; if it ever becomes too big we can erase it with: ``` mc rm --recursive --force 'garage/nix/' @@ -157,9 +136,9 @@ nix-shell --run refresh_index If you want to compile for different architectures, you will need to repeat all these commands for each architecture. -**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone** +**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.** -### Drone +### Drone (obsolete) Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr). You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it. diff --git a/doc/book/operations/durability-repairs.md b/doc/book/operations/durability-repairs.md index b0d2c78a..fdf163e2 100644 --- a/doc/book/operations/durability-repairs.md +++ b/doc/book/operations/durability-repairs.md @@ -19,7 +19,7 @@ connecting to. To run on all nodes, add the `-a` flag as follows: # Data block operations -## Data store scrub +## Data store scrub {#scrub} Scrubbing the data store means examining each individual data block to check that their content is correct, by verifying their hash. Any block found to be corrupted @@ -49,7 +49,7 @@ verifications. Of course, scrubbing the entire data store will also take longer. ## Block check and resync In some cases, nodes hold a reference to a block but do not actually have the block -stored on disk. Conversely, they may also have on disk blocks that are not referenced +stored on disk. Conversely, they may also have on-disk blocks that are not referenced any more. To fix both cases, a block repair may be run with `garage repair blocks`. This will scan the entire block reference counter table to check that the blocks exist on disk, and will scan the entire disk store to check that stored blocks @@ -95,7 +95,7 @@ using the `garage block purge` command. In [multi-HDD setups](@/documentation/operations/multi-hdd.md), to ensure that data blocks are well balanced between storage locations, you may run a -rebalance operation using `garage repair rebalance`. This is usefull when +rebalance operation using `garage repair rebalance`. This is useful when adding storage locations or when capacities of the storage locations have been changed. Once this is finished, Garage will know for each block of a single possible location where it can be, which can increase access speed. This @@ -104,6 +104,24 @@ operation will also move out all data from locations marked as read-only. # Metadata operations +## Metadata snapshotting + +It is good practice to setup automatic snapshotting of your metadata database +file, to recover from situations where it becomes corrupted on disk. This can +be done at the filesystem level if you are using ZFS or BTRFS. + +Since Garage v0.9.4, Garage is able to take snapshots of the metadata database +itself. This basically amounts to copying the database file, except that it can +be run live while Garage is running without the risk of corruption or +inconsistencies. This can be setup to run automatically on a schedule using +[`metadata_auto_snapshot_interval`](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval). +A snapshot can also be triggered manually using the `garage meta snapshot` +command. Note that taking a snapshot using this method is very intensive as it +requires making a full copy of the database file, so you might prefer using +filesystem-level snapshots if possible. To recover a corrupted node from such a +snapshot, read the instructions +[here](@/documentation/operations/recovering.md#corrupted_meta). + ## Metadata table resync Garage automatically resyncs all entries stored in the metadata tables every hour, @@ -123,4 +141,7 @@ blocks may still be held by Garage. If you suspect that such corruption has occu in your cluster, you can run one of the following repair procedures: - `garage repair versions`: checks that all versions belong to a non-deleted object, and purges any orphan version -- `garage repair block_refs`: checks that all block references belong to a non-deleted object version, and purges any orphan block reference (this will then allow the blocks to be garbage-collected) + +- `garage repair block-refs`: checks that all block references belong to a non-deleted object version, and purges any orphan block reference (this will then allow the blocks to be garbage-collected) + +- `garage repair block-rc`: checks that the reference counters for blocks are in sync with the actual number of non-deleted entries in the block reference table diff --git a/doc/book/operations/layout.md b/doc/book/operations/layout.md index ece17ddb..667e89d2 100644 --- a/doc/book/operations/layout.md +++ b/doc/book/operations/layout.md @@ -12,8 +12,8 @@ An introduction to building cluster layouts can be found in the [production depl In Garage, all of the data that can be stored in a given cluster is divided into slices which we call *partitions*. Each partition is stored by one or several nodes in the cluster -(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication-mode)). -The layout determines the correspondence between these partition, +(see [`replication_factor`](@/documentation/reference-manual/configuration.md#replication_factor)). +The layout determines the correspondence between these partitions, which exist on a logical level, and actual storage nodes. ## How cluster layouts work in Garage @@ -94,10 +94,10 @@ follow the following recommendations: ## Understanding unexpected layout calculations When adding, removing or modifying nodes in a cluster layout, sometimes -unexpected assigntations of partitions to node can occur. These assignations -are in fact normal and logical, given the objectives of the algorihtm. Indeed, -**the layout algorithm prioritizes moving less data between nodes over the fact -of achieving equal distribution of load. It also tries to use all links between +unexpected assignations of partitions to node can occur. These assignations +are in fact normal and logical, given the objectives of the algorithm. Indeed, +**the layout algorithm prioritizes moving less data between nodes over +achieving equal distribution of load. It also tries to use all links between pairs of nodes in equal proportions when moving data.** This section presents two examples and illustrates how one can control Garage's behavior to obtain the desired results. @@ -270,5 +270,5 @@ that is moved to node1). This illustrates the second principle of the layout computation: **if there is a choice in moving data out of some nodes, then all links between pairs of nodes are used in equal proportions** (this is approximately true, there is -randomness in the algorihtm to achieve this so there might be some small +randomness in the algorithm to achieve this so there might be some small fluctuations, as we see above). diff --git a/doc/book/operations/multi-hdd.md b/doc/book/operations/multi-hdd.md index 36445b0a..1cbcd805 100644 --- a/doc/book/operations/multi-hdd.md +++ b/doc/book/operations/multi-hdd.md @@ -21,14 +21,14 @@ data_dir = [ ``` Garage will automatically balance all blocks stored by the node -among the different specified directories, proportionnally to the +among the different specified directories, proportionally to the specified capacities. ## Updating the list of storage locations If you add new storage locations to your `data_dir`, Garage will not rebalance existing data between storage locations. -Newly written blocks will be balanced proportionnally to the specified capacities, +Newly written blocks will be balanced proportionally to the specified capacities, and existing data may be moved between drives to improve balancing, but only opportunistically when a data block is re-written (e.g. an object is re-uploaded, or an object with a duplicate block is uploaded). diff --git a/doc/book/operations/recovering.md b/doc/book/operations/recovering.md index 7a830788..05322b67 100644 --- a/doc/book/operations/recovering.md +++ b/doc/book/operations/recovering.md @@ -5,7 +5,7 @@ weight = 40 Garage is meant to work on old, second-hand hardware. In particular, this makes it likely that some of your drives will fail, and some manual intervention will be needed. -Fear not! For Garage is fully equipped to handle drive failures, in most common cases. +Fear not! Garage is fully equipped to handle drive failures, in most common cases. ## A note on availability of Garage @@ -61,7 +61,7 @@ garage repair -a --yes blocks This will re-synchronize blocks of data that are missing to the new HDD, reading them from copies located on other nodes. -You can check on the advancement of this process by doing the following command: +You can check on the advancement of this process by doing the following command: ```bash garage stats -a @@ -108,3 +108,57 @@ garage layout apply # once satisfied, apply the changes Garage will then start synchronizing all required data on the new node. This process can be monitored using the `garage stats -a` command. + +## Replacement scenario 3: corrupted metadata {#corrupted_meta} + +In some cases, your metadata DB file might become corrupted, for instance if +your node suffered a power outage and did not shut down properly. In this case, +you can recover without having to change the node ID and rebuilding a cluster +layout. This means that data blocks will not need to be shuffled around, you +must simply find a way to repair the metadata file. The best way is generally +to discard the corrupted file and recover it from another source. + +First of all, start by locating the database file in your metadata directory, +which [depends on your `db_engine` +choice](@/documentation/reference-manual/configuration.md#db_engine). Then, +your recovery options are as follows: + +- **Option 1: resyncing from other nodes.** In case your cluster is replicated + with two or three copies, you can simply delete the database file, and Garage + will resync from other nodes. To do so, stop Garage, delete the database file + or directory, and restart Garage. Then, do a full table repair by calling + `garage repair -a --yes tables`. This will take a bit of time to complete as + the new node will need to receive copies of the metadata tables from the + network. + +- **Option 2: restoring a snapshot taken by Garage.** Since v0.9.4, Garage can + [automatically take regular + snapshots](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval) + of your metadata DB file. This file or directory should be located under + `/snapshots`, and is named according to the UTC time at which it + was taken. Stop Garage, discard the database file/directory and replace it by the + snapshot you want to use. For instance, in the case of LMDB: + + ```bash + cd $METADATA_DIR + mv db.lmdb db.lmdb.bak + cp -r snapshots/2024-03-15T12:13:52Z db.lmdb + ``` + + And for Sqlite: + + ```bash + cd $METADATA_DIR + mv db.sqlite db.sqlite.bak + cp snapshots/2024-03-15T12:13:52Z db.sqlite + ``` + + Then, restart Garage and run a full table repair by calling `garage repair -a + --yes tables`. This should run relatively fast as only the changes that + occurred since the snapshot was taken will need to be resynchronized. Of + course, if your cluster is not replicated, you will lose all changes that + occurred since the snapshot was taken. + +- **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or + BTRFS to snapshot your metadata partition, refer to their specific + documentation on rolling back or copying files from an old snapshot. diff --git a/doc/book/operations/upgrading.md b/doc/book/operations/upgrading.md index 9a738282..a3d2bcf5 100644 --- a/doc/book/operations/upgrading.md +++ b/doc/book/operations/upgrading.md @@ -9,7 +9,7 @@ On a new version release, there is 2 possibilities: - protocols and data structures remained the same ➡️ this is a **minor upgrade** - protocols or data structures changed ➡️ this is a **major upgrade** -You can quickly now what type of update you will have to operate by looking at the version identifier: +You can quickly know what type of update you will have to operate by looking at the version identifier: when we require our users to do a major upgrade, we will always bump the first nonzero component of the version identifier (e.g. from v0.7.2 to v0.8.0). Conversely, for versions that only require a minor upgrade, the first nonzero component will always stay the same (e.g. from v0.8.0 to v0.8.1). @@ -71,7 +71,19 @@ The entire procedure would look something like this: 2. Take each node offline individually to back up its metadata folder, bring them back online once the backup is done. You can do all of the nodes in a single zone at once as that won't impact global cluster availability. - Do not try to make a backup of the metadata folder of a running node. + Do not try to manually copy the metadata folder of a running node. + + **Since Garage v0.9.4,** you can use the `garage meta snapshot --all` command + to take a simultaneous snapshot of the metadata database files of all your + nodes. This avoids the tedious process of having to take them down one by + one before upgrading. Be careful that if automatic snapshotting is enabled, + Garage only keeps the last two snapshots and deletes older ones, so you might + want to disable automatic snapshotting in your upgraded configuration file + until you have confirmed that the upgrade ran successfully. In addition to + snapshotting the metadata databases of your nodes, you should back-up at + least the `cluster_layout` file of one of your Garage instances (this file + should be the same on all nodes and you can copy it safely while Garage is + running). 3. Prepare your binaries and configuration files for the new Garage version diff --git a/doc/book/quick-start/_index.md b/doc/book/quick-start/_index.md index bd64e3eb..633b785a 100644 --- a/doc/book/quick-start/_index.md +++ b/doc/book/quick-start/_index.md @@ -42,6 +42,13 @@ If a binary of the last version is not available for your architecture, or if you want a build customized for your system, you can [build Garage from source](@/documentation/cookbook/from-source.md). +If none of these option work for you, you can also run Garage in a Docker +container. When using Docker, the commands used in this guide will not work +anymore. We recommend reading the tutorial on [configuring a +multi-node cluster](@/documentation/cookbook/real-world.md) to learn about +using Garage as a Docker container. For simplicity, a minimal command to launch +Garage using Docker is provided in this quick start guide as well. + ## Configuring and starting Garage @@ -57,9 +64,9 @@ to generate unique and private secrets for security reasons: cat > garage.toml < +garage layout assign -z dc1 -c 1G ``` where `` corresponds to the identifier of the node shown by `garage status` (first column). You can enter simply a prefix of that identifier. -For instance here you could write just `garage layout assign -z dc1 -c 1 563e`. +For instance here you could write just `garage layout assign -z dc1 -c 1G 563e`. The layout then has to be applied to the cluster, using: ```bash -garage layout apply +garage layout apply --version 1 ``` @@ -248,7 +285,7 @@ garage bucket info nextcloud-bucket ``` -## Uploading and downlading from Garage +## Uploading and downloading from Garage To download and upload files on garage, we can use a third-party tool named `awscli`. @@ -269,12 +306,14 @@ named `~/.awsrc` with this content: export AWS_ACCESS_KEY_ID=xxxx # put your Key ID here export AWS_SECRET_ACCESS_KEY=xxxx # put your Secret key here export AWS_DEFAULT_REGION='garage' -export AWS_ENDPOINT='http://localhost:3900' +export AWS_ENDPOINT_URL='http://localhost:3900' -function aws { command aws --endpoint-url $AWS_ENDPOINT $@ ; } aws --version ``` +Note you need to have at least `awscli` `>=1.29.0` or `>=2.13.0`, otherwise you +need to specify `--endpoint-url` explicitly on each `awscli` invocation. + Now, each time you want to use `awscli` on this target, run: ```bash @@ -311,7 +350,7 @@ Check [our s3 compatibility list](@/documentation/reference-manual/s3-compatibil ### Other tools for interacting with Garage -The following tools can also be used to send and recieve files from/to Garage: +The following tools can also be used to send and receive files from/to Garage: - [minio-client](@/documentation/connect/cli.md#minio-client) - [s3cmd](@/documentation/connect/cli.md#s3cmd) diff --git a/doc/book/reference-manual/admin-api.md b/doc/book/reference-manual/admin-api.md index 6932ac60..fcf49e8c 100644 --- a/doc/book/reference-manual/admin-api.md +++ b/doc/book/reference-manual/admin-api.md @@ -8,18 +8,21 @@ listen address is specified in the `[admin]` section of the configuration file (see [configuration file reference](@/documentation/reference-manual/configuration.md)) -**WARNING.** At this point, there is no comittement to stability of the APIs described in this document. -We will bump the version numbers prefixed to each API endpoint at each time the syntax +**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document. +We will bump the version numbers prefixed to each API endpoint each time the syntax or semantics change, meaning that code that relies on these endpoint will break when changes are introduced. -The Garage administration API was introduced in version 0.7.2, this document -does not apply to older versions of Garage. +Versions: + - Before Garage 0.7.2 - no admin API + - Garage 0.7.2 - admin APIv0 + - Garage 0.9.0 - admin APIv1, deprecate admin APIv0 + ## Access control -The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: +The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without @@ -85,8 +88,8 @@ Consult the full health check API endpoint at /v0/health for more details ### On-demand TLS `GET /check` -To prevent abuses for on-demand TLS, Caddy developpers have specified an endpoint that can be queried by the reverse proxy -to know if a given domain is allowed to get a certificate. Garage implements this endpoints to tell if a given domain is handled by Garage or is garbage. +To prevent abuse for on-demand TLS, Caddy developers have specified an endpoint that can be queried by the reverse proxy +to know if a given domain is allowed to get a certificate. Garage implements these endpoints to tell if a given domain is handled by Garage or is garbage. Garage responds with the following logic: - If the domain matches the pattern `.`, returns 200 OK @@ -99,7 +102,7 @@ You must manually declare the domain in your reverse-proxy. Idem for K2V.* *Note 2: buckets in a user's namespace are not supported yet by this endpoint. This is a limitation of this endpoint currently.* -**Example:** Suppose a Garage instance configured with `s3_api.root_domain = .s3.garage.localhost` and `s3_web.root_domain = .web.garage.localhost`. +**Example:** Suppose a Garage instance is configured with `s3_api.root_domain = .s3.garage.localhost` and `s3_web.root_domain = .web.garage.localhost`. With a private `media` bucket (name in the global namespace, website is disabled), the endpoint will feature the following behavior: @@ -131,7 +134,9 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl ### Cluster operations -These endpoints are defined on a dedicated [Redocly page](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html). You can also download its [OpenAPI specification](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml). +These endpoints have a dedicated OpenAPI spec. + - APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml) + - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml) Requesting the API from the command line can be as simple as running: diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index f07fb1e0..1f583fe6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -8,34 +8,46 @@ weight = 20 Here is an example `garage.toml` configuration file that illustrates all of the possible options: ```toml +replication_factor = 3 +consistency_mode = "consistent" + metadata_dir = "/var/lib/garage/meta" data_dir = "/var/lib/garage/data" +metadata_snapshots_dir = "/var/lib/garage/snapshots" metadata_fsync = true data_fsync = false +disable_scrub = false +use_local_tz = false +metadata_auto_snapshot_interval = "6h" db_engine = "lmdb" -block_size = 1048576 - -sled_cache_capacity = "128MiB" -sled_flush_every_ms = 2000 +block_size = "1M" +block_ram_buffer_max = "256MiB" +block_max_concurrent_reads = 16 +block_max_concurrent_writes_per_request =10 lmdb_map_size = "1T" -replication_mode = "3" - compression_level = 1 rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6" rpc_bind_addr = "[::]:3901" +rpc_bind_outgoing = false rpc_public_addr = "[fc00:1::1]:3901" +# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet: +# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64" + + +allow_world_readable_secrets = false bootstrap_peers = [ "563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901", - "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332[fc00:1::2]:3901", + "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901", "681456ab91350f92242e80a531a3ec9392cb7c974f72640112f90a600d7921a4@[fc00:B::1]:3901", "212fd62eeaca72c122b45a7f4fa0f55e012aa5e24ac384a72a3016413fa724ff@[fc00:F::1]:3901", ] +allow_punycode = false [consul_discovery] api = "catalog" @@ -65,11 +77,12 @@ root_domain = ".s3.garage" [s3_web] bind_addr = "[::]:3902" root_domain = ".web.garage" +add_host_to_metrics = true [admin] api_bind_addr = "0.0.0.0:3903" -metrics_token = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81" -admin_token = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a" +metrics_token = "BCAdFjoa9G0KJR0WXnHHm7fs1ZAbfpI8iIZ+Z/a2NgI=" +admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=" trace_sink = "http://localhost:4317" ``` @@ -77,7 +90,197 @@ The following gives details about each available configuration option. ## Available configuration options -### `metadata_dir` +### Index + +[Environment variables](#env_variables). + +Top-level configuration options, in alphabetical order: +[`allow_punycode`](#allow_punycode), +[`allow_world_readable_secrets`](#allow_world_readable_secrets), +[`block_max_concurrent_reads`](`block_max_concurrent_reads), +[`block_ram_buffer_max`](#block_ram_buffer_max), +[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request), +[`block_size`](#block_size), +[`bootstrap_peers`](#bootstrap_peers), +[`compression_level`](#compression_level), +[`consistency_mode`](#consistency_mode), +[`data_dir`](#data_dir), +[`data_fsync`](#data_fsync), +[`db_engine`](#db_engine), +[`disable_scrub`](#disable_scrub), +[`lmdb_map_size`](#lmdb_map_size), +[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval), +[`metadata_dir`](#metadata_dir), +[`metadata_fsync`](#metadata_fsync), +[`metadata_snapshots_dir`](#metadata_snapshots_dir), +[`replication_factor`](#replication_factor), +[`rpc_bind_addr`](#rpc_bind_addr), +[`rpc_bind_outgoing`](#rpc_bind_outgoing), +[`rpc_public_addr`](#rpc_public_addr), +[`rpc_public_addr_subnet`](#rpc_public_addr_subnet) +[`rpc_secret`/`rpc_secret_file`](#rpc_secret), +[`use_local_tz`](#use_local_tz). + +The `[consul_discovery]` section: +[`api`](#consul_api), +[`ca_cert`](#consul_ca_cert), +[`client_cert`](#consul_client_cert_and_key), +[`client_key`](#consul_client_cert_and_key), +[`consul_http_addr`](#consul_http_addr), +[`meta`](#consul_tags_and_meta), +[`service_name`](#consul_service_name), +[`tags`](#consul_tags_and_meta), +[`tls_skip_verify`](#consul_tls_skip_verify), +[`token`](#consul_token). + +The `[kubernetes_discovery]` section: +[`namespace`](#kube_namespace), +[`service_name`](#kube_service_name), +[`skip_crd`](#kube_skip_crd). + +The `[s3_api]` section: +[`api_bind_addr`](#s3_api_bind_addr), +[`root_domain`](#s3_root_domain), +[`s3_region`](#s3_region). + +The `[s3_web]` section: +[`add_host_to_metrics`](#web_add_host_to_metrics), +[`bind_addr`](#web_bind_addr), +[`root_domain`](#web_root_domain). + +The `[admin]` section: +[`api_bind_addr`](#admin_api_bind_addr), +[`metrics_token`/`metrics_token_file`](#admin_metrics_token), +[`admin_token`/`admin_token_file`](#admin_token), +[`trace_sink`](#admin_trace_sink), + +### Environment variables {#env_variables} + +The following configuration parameters must be specified as environment variables, +they do not exist in the configuration file: + +- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the + Garage daemon send its logs to `syslog` (using the libc `syslog` function) + instead of printing to stderr. + +- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the + Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`) + instead of printing to stderr. + +The following environment variables can be used to override the corresponding +values in the configuration file: + +- [`GARAGE_ALLOW_WORLD_READABLE_SECRETS`](#allow_world_readable_secrets) +- [`GARAGE_RPC_SECRET` and `GARAGE_RPC_SECRET_FILE`](#rpc_secret) +- [`GARAGE_ADMIN_TOKEN` and `GARAGE_ADMIN_TOKEN_FILE`](#admin_token) +- [`GARAGE_METRICS_TOKEN` and `GARAGE_METRICS_TOKEN`](#admin_metrics_token) + + +### Top-level configuration options + +#### `replication_factor` (since `v1.0.0`) {#replication_factor} + +The replication factor can be any positive integer smaller or equal the node count in your cluster. +The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics. + +- `1`: data stored on Garage is stored on a single node. There is no + redundancy, and data will be unavailable as soon as one node fails or its + network is disconnected. Do not use this for anything else than test + deployments. + +- `2`: data stored on Garage will be stored on two different nodes, if possible + in different zones. Garage tolerates one node failure, or several nodes + failing but all in a single zone (in a deployment with at least two zones), + before losing data. Data remains available in read-only mode when one node is + down, but write operations will fail. + +- `3`: data stored on Garage will be stored on three different nodes, if + possible each in a different zones. Garage tolerates two node failure, or + several node failures but in no more than two zones (in a deployment with at + least three zones), before losing data. As long as only a single node fails, + or node failures are only in a single zone, reading and writing data to + Garage can continue normally. + +- `5`, `7`, ...: When setting the replication factor above 3, it is most useful to + choose an uneven value, since for every two copies added, one more node can fail + before losing the ability to write and read to the cluster. + +Note that in modes `2` and `3`, +if at least the same number of zones are available, an arbitrary number of failures in +any given zone is tolerated as copies of data will be spread over several zones. + +**Make sure `replication_factor` is the same in the configuration files of all nodes. +Never run a Garage cluster where that is not the case.** + +It is technically possible to change the replication factor although it's a +dangerous operation that is not officially supported. This requires you to +delete the existing cluster layout and create a new layout from scratch, +meaning that a full rebalancing of your cluster's data will be needed. To do +it, shut down your cluster entirely, delete the `custer_layout` files in the +meta directories of all your nodes, update all your configuration files with +the new `replication_factor` parameter, restart your cluster, and then create a +new layout with all the nodes you want to keep. Rebalancing data will take +some time, and data might temporarily appear unavailable to your users. +It is recommended to shut down public access to the cluster while rebalancing +is in progress. In theory, no data should be lost as rebalancing is a +routine operation for Garage, although we cannot guarantee you that everything + will go right in such an extreme scenario. + +#### `consistency_mode` (since `v1.0.0`) {#consistency_mode} + +The consistency mode setting determines the read and write behaviour of your cluster. + + - `consistent`: The default setting. This is what the paragraph above describes. + The read and write quorum will be determined so that read-after-write consistency + is guaranteed. + - `degraded`: Lowers the read + quorum to `1`, to allow you to read data from your cluster when several + nodes (or nodes in several zones) are unavailable. In this mode, Garage + does not provide read-after-write consistency anymore. + The write quorum stays the same as in the `consistent` mode, ensuring that + data successfully written to Garage is stored on multiple nodes (depending + the replication factor). + - `dangerous`: This mode lowers both the read + and write quorums to `1`, to allow you to both read and write to your + cluster when several nodes (or nodes in several zones) are unavailable. It + is the least consistent mode of operation proposed by Garage, and also one + that should probably never be used. + +Changing the `consistency_mode` between modes while leaving the `replication_factor` untouched +(e.g. setting your node's `consistency_mode` to `degraded` when it was previously unset, or from +`dangerous` to `consistent`), can be done easily by just changing the `consistency_mode` +parameter in your config files and restarting all your Garage nodes. + +The consistency mode can be used together with various replication factors, to achieve +a wide range of read and write characteristics. Some examples: + + - Replication factor `2`, consistency mode `degraded`: While this mode + technically exists, its properties are the same as with consistency mode `consistent`, + since the read quorum with replication factor `2`, consistency mode `consistent` is already 1. + + - Replication factor `2`, consistency mode `dangerous`: written objects are written to + the second replica asynchronously. This means that Garage will return `200 + OK` to a PutObject request before the second copy is fully written (or even + before it even starts being written). This means that data can more easily + be lost if the node crashes before a second copy can be completed. This + also means that written objects might not be visible immediately in read + operations. In other words, this configuration severely breaks the consistency and + durability guarantees of standard Garage cluster operation. Benefits of + this configuration: you can still write to your cluster when one node is + unavailable. + +The quorums associated with each replication mode are described below: + +| `consistency_mode` | `replication_factor` | Write quorum | Read quorum | Read-after-write consistency? | +| ------------------ | -------------------- | ------------ | ----------- | ----------------------------- | +| `consistent` | 1 | 1 | 1 | yes | +| `consistent` | 2 | 2 | 1 | yes | +| `dangerous` | 2 | 1 | 1 | NO | +| `consistent` | 3 | 2 | 2 | yes | +| `degraded` | 3 | 2 | 1 | NO | +| `dangerous` | 3 | 1 | 1 | NO | + +#### `metadata_dir` {#metadata_dir} The directory in which Garage will store its metadata. This contains the node identifier, the network configuration and the peer list, the list of buckets and keys as well @@ -85,7 +288,8 @@ as the index of all objects, object version and object blocks. Store this folder on a fast SSD drive if possible to maximize Garage's performance. -### `data_dir` + +#### `data_dir` {#data_dir} The directory in which Garage will store the data blocks of objects. This folder can be placed on an HDD. The space available for `data_dir` @@ -105,48 +309,90 @@ data_dir = [ See [the dedicated documentation page](@/documentation/operations/multi-hdd.md) on how to operate Garage in such a setup. -### `db_engine` (since `v0.8.0`) +#### `metadata_snapshots_dir` (since `v1.1.0`) {#metadata_snapshots_dir} -By default, Garage uses the Sled embedded database library -to store its metadata on-disk. Since `v0.8.0`, Garage can use alternative storage backends as follows: +The directory in which Garage will store metadata snapshots when it +performs a snapshot of the metadata database, either when instructed to do +so from a RPC call or regularly through +[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval). + +By default, Garage will store snapshots into a `snapshots/` subdirectory +of [`metadata_dir`](#metadata_dir). This might quickly fill up your +metadata storage space if you use snapshots, because Garage will need up +to 4x the space of the existing metadata database: each snapshot requires +roughly as much space as the original database, and Garage temporarily +needs to store up to three different snapshots before it cleans up the oldest +snapshot to go back to two stored snapshots. + +To prevent filling your disk, you might to change this setting to a +directory with ample available space, e.g. on the same storage space as +[`data_dir`](#data_dir). + +#### `db_engine` (since `v0.8.0`) {#db_engine} + +Since `v0.8.0`, Garage can use alternative storage backends as follows: | DB engine | `db_engine` value | Database path | | --------- | ----------------- | ------------- | -| [Sled](https://sled.rs) | `"sled"` | `/db/` | -| [LMDB](https://www.lmdb.tech) | `"lmdb"` | `/db.lmdb/` | -| [Sqlite](https://sqlite.org) | `"sqlite"` | `/db.sqlite` | +| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `/db.lmdb/` | +| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `/db.sqlite` | +| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `/db.fjall/` | +| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `/db/` | + +Sled was supported until Garage v0.9.x, and was removed in Garage v1.0. +You can still use an older binary of Garage (e.g. v0.9.4) to migrate +old Sled metadata databases to another engine. Performance characteristics of the different DB engines are as follows: -- Sled: the default database engine, which tends to produce - large data files and also has performance issues, especially when the metadata folder - is on a traditional HDD and not on SSD. -- LMDB: the recommended alternative on 64-bit systems, - much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable - between architectures, so for instance the Garage database of an x86-64 - node cannot be moved to an ARM64 node. Also note that, while LMDB can technically be used on 32-bit systems, - this will limit your node to very small database sizes due to how LMDB works; it is therefore not recommended. -- Sqlite: Garage supports Sqlite as a storage backend for metadata, - however it may have issues and is also very slow in its current implementation, - so it is not recommended to be used for now. +- LMDB: the recommended database engine for high-performance distributed clusters. +LMDB works very well, but is known to have the following limitations: -It is possible to convert Garage's metadata directory from one format to another with a small utility named `convert_db`, -which can be downloaded at the following locations: -[for amd64](https://garagehq.deuxfleurs.fr/_releases/convert_db/amd64/convert_db), -[for i386](https://garagehq.deuxfleurs.fr/_releases/convert_db/i386/convert_db), -[for arm64](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm64/convert_db), -[for arm](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm/convert_db). -The `convert_db` utility is used as folows: + - The data format of LMDB is not portable between architectures, so for + instance the Garage database of an x86-64 node cannot be moved to an ARM64 + node. + + - While LMDB can technically be used on 32-bit systems, this will limit your + node to very small database sizes due to how LMDB works; it is therefore + not recommended. + + - Several users have reported corrupted LMDB database files after an unclean + shutdown (e.g. a power outage). This situation can generally be recovered + from if your cluster is geo-replicated (by rebuilding your metadata db from + other nodes), or if you have saved regular snapshots at the filesystem + level. + + - Keys in LMDB are limited to 511 bytes. This limit translates to limits on + object keys in S3 and sort keys in K2V that are limted to 479 bytes. + +- Sqlite: Garage supports Sqlite as an alternative storage backend for + metadata, which does not have the issues listed above for LMDB. + On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal + performance, which was fixed with the addition of `metadata_fsync`. + Sqlite is still probably slower than LMDB due to the way we use it, + so it is not the best choice for high-performance storage clusters, + but it should work fine in many cases. + +- Fjall: a storage engine based on LSM trees, which theoretically allow for + higher write throughput than other storage engines that are based on B-trees. + Using Fjall could potentially improve Garage's performance significantly in + write-heavy workloads. **Support for Fjall is experimental at this point**, + we have added it to Garage for evaluation purposes only. **Do not use it for + production-critical workloads.** + + +It is possible to convert Garage's metadata directory from one format to another +using the `garage convert-db` command, which should be used as follows: ``` -convert-db -a -i \ - -b -o +garage convert-db -a -i \ + -b -o ``` -Make sure to specify the full database path as presented in the table above, -and not just the path to the metadata directory. +Make sure to specify the full database path as presented in the table above +(third colummn), and not just the path to the metadata directory. -### `metadata_fsync` +#### `metadata_fsync` {#metadata_fsync} Whether to enable synchronous mode for the database engine or not. This is disabled (`false`) by default. @@ -160,7 +406,7 @@ Using this option reduces the risk of simultaneous metadata corruption on severa cluster nodes, which could lead to data loss. If multi-site replication is used, this option is most likely not necessary, as -it is extremely unlikely that two nodes in different locations will have a +it is extremely unlikely that two nodes in different locations will have a power failure at the exact same time. (Metadata corruption on a single node is not an issue, the corrupted data file @@ -170,13 +416,13 @@ Here is how this option impacts the different database engines: | Database | `metadata_fsync = false` (default) | `metadata_fsync = true` | |----------|------------------------------------|-------------------------------| -| Sled | default options | *unsupported* | | Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` | | LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` | +| Fjall | default options | not supported | Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`). -### `data_fsync` +#### `data_fsync` {#data_fsync} Whether to `fsync` data blocks and their containing directory after they are saved to disk. @@ -189,7 +435,51 @@ at the cost of a moderate drop in write performance. Similarly to `metatada_fsync`, this is likely not necessary if geographical replication is used. -### `block_size` +#### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval} + +If this value is set, Garage will automatically take a snapshot of the metadata +DB file at a regular interval and save it in the metadata directory. +This parameter can take any duration string that can be parsed by +the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate. + +Snapshots can allow to recover from situations where the metadata DB file is +corrupted, for instance after an unclean shutdown. See [this +page](@/documentation/operations/recovering.md#corrupted_meta) for details. +Garage keeps only the two most recent snapshots of the metadata DB and deletes +older ones automatically. + +Note that taking a metadata snapshot is a relatively intensive operation as the +entire data file is copied. A snapshot being taken might have performance +impacts on the Garage node while it is running. If the cluster is under heavy +write load when a snapshot operation is running, this might also cause the +database file to grow in size significantly as pages cannot be recycled easily. +For this reason, it might be better to use filesystem-level snapshots instead +if possible. + +#### `disable_scrub` {#disable_scrub} + +By default, Garage runs a scrub of the data directory approximately once per +month, with a random delay to avoid all nodes running at the same time. When +it scrubs the data directory, Garage will read all of the data files stored on +disk to check their integrity, and will rebuild any data files that it finds +corrupted, using the remaining valid copies stored on other nodes. +See [this page](@/documentation/operations/durability-repairs.md#scrub) for details. + +Set the `disable_scrub` configuration value to `true` if you don't need Garage +to scrub the data directory, for instance if you are already scrubbing at the +filesystem level. Note that in this case, if you find a corrupted data file, +you should delete it from the data directory and then call `garage repair +blocks` on the node to ensure that it re-obtains a copy from another node on +the network. + +#### `use_local_tz` (since `v1.1.0`) {#use_local_tz} + +By default, Garage runs the lifecycle worker every day at midnight in UTC. Set the +`use_local_tz` configuration value to `true` if you want Garage to run the +lifecycle worker at midnight in your local timezone. If you have multiple nodes, +you should also ensure that each node has the same timezone configuration. + +#### `block_size` {#block_size} Garage splits stored objects in consecutive chunks of size `block_size` (except the last one which might be smaller). The default size is 1MiB and @@ -204,22 +494,69 @@ files will remain available. This however means that chunks from existing files will not be deduplicated with chunks from newly uploaded files, meaning you might use more storage space that is optimally possible. -### `sled_cache_capacity` +#### `block_ram_buffer_max` (since `v0.9.4`) {#block_ram_buffer_max} -This parameter can be used to tune the capacity of the cache used by -[sled](https://sled.rs), the database Garage uses internally to store metadata. -Tune this to fit the RAM you wish to make available to your Garage instance. -This value has a conservative default (128MB) so that Garage doesn't use too much -RAM by default, but feel free to increase this for higher performance. +A limit on the total size of data blocks kept in RAM by S3 API nodes awaiting +to be sent to storage nodes asynchronously. -### `sled_flush_every_ms` +Explanation: since Garage wants to tolerate node failures, it uses quorum +writes to send data blocks to storage nodes: try to write the block to three +nodes, and return ok as soon as two writes complete. So even if all three nodes +are online, the third write always completes asynchronously. In general, there +are not many writes to a cluster, and the third asynchronous write can +terminate early enough so as to not cause unbounded RAM growth. However, if +the S3 API node is continuously receiving large quantities of data and the +third node is never able to catch up, many data blocks will be kept buffered in +RAM as they are awaiting transfer to the third node. -This parameters can be used to tune the flushing interval of sled. -Increase this if sled is thrashing your SSD, at the risk of losing more data in case -of a power outage (though this should not matter much as data is replicated on other -nodes). The default value, 2000ms, should be appropriate for most use cases. +The `block_ram_buffer_max` sets a limit to the size of buffers that can be kept +in RAM in this process. When the limit is reached, backpressure is applied +back to the S3 client. -### `lmdb_map_size` +Note that this only counts buffers that have arrived to a certain stage of +processing (received from the client + encrypted and/or compressed as +necessary) and are ready to send to the storage nodes. Many other buffers will +not be counted and this is not a hard limit on RAM consumption. In particular, +if many clients send requests simultaneously with large objects, the RAM +consumption will always grow linearly with the number of concurrent requests, +as each request will use a few buffers of size `block_size` for receiving and +intermediate processing before even trying to send the data to the storage +node. + +The default value is 256MiB. + +#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads} + +The maximum number of blocks (individual files in the data directory) open +simultaneously for reading. + +Reducing this number does not limit the number of data blocks that can be +transferred through the network simultaneously. This mechanism was just added +as a backpressure mechanism for HDD read speed: it helps avoid a situation +where too many requests are coming in and Garage is reading too many block +files simultaneously, thus not making timely progress on any of the reads. + +When a request to read a data block comes in through the network, the requests +awaits for one of the `block_max_concurrent_reads` slots to be available +(internally implemented using a Semaphore object). Once it acquired a read +slot, it reads the entire block file to RAM and frees the slot as soon as the +block file is finished reading. Only after the slot is released will the +block's data start being transferred over the network. If the request fails to +acquire a reading slot wihtin 15 seconds, it fails with a timeout error. +Timeout events can be monitored through the `block_read_semaphore_timeouts` +metric in Prometheus: a non-zero number of such events indicates an I/O +bottleneck on HDD read speed. + + +#### `block_max_concurrent_writes_per_request` (since `v2.1.0`) {#block_max_concurrent_writes_per_request} + +This parameter is designed to adapt to the concurrent write performance of +different storage media.Maximum number of parallel block writes per put request +Higher values improve throughput but increase memory usage. + +Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + +#### `lmdb_map_size` {#lmdb_map_size} This parameters can be used to set the map size used by LMDB, which is the size of the virtual memory region used for mapping the database file. @@ -227,90 +564,7 @@ The value of this parameter is the maximum size the metadata database can take. This value is not bound by the physical RAM size of the machine running Garage. If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines. -### `replication_mode` - -Garage supports the following replication modes: - -- `none` or `1`: data stored on Garage is stored on a single node. There is no - redundancy, and data will be unavailable as soon as one node fails or its - network is disconnected. Do not use this for anything else than test - deployments. - -- `2`: data stored on Garage will be stored on two different nodes, if possible - in different zones. Garage tolerates one node failure, or several nodes - failing but all in a single zone (in a deployment with at least two zones), - before losing data. Data remains available in read-only mode when one node is - down, but write operations will fail. - - - `2-dangerous`: a variant of mode `2`, where written objects are written to - the second replica asynchronously. This means that Garage will return `200 - OK` to a PutObject request before the second copy is fully written (or even - before it even starts being written). This means that data can more easily - be lost if the node crashes before a second copy can be completed. This - also means that written objects might not be visible immediately in read - operations. In other words, this mode severely breaks the consistency and - durability guarantees of standard Garage cluster operation. Benefits of - this mode: you can still write to your cluster when one node is - unavailable. - -- `3`: data stored on Garage will be stored on three different nodes, if - possible each in a different zones. Garage tolerates two node failure, or - several node failures but in no more than two zones (in a deployment with at - least three zones), before losing data. As long as only a single node fails, - or node failures are only in a single zone, reading and writing data to - Garage can continue normally. - - - `3-degraded`: a variant of replication mode `3`, that lowers the read - quorum to `1`, to allow you to read data from your cluster when several - nodes (or nodes in several zones) are unavailable. In this mode, Garage - does not provide read-after-write consistency anymore. The write quorum is - still 2, ensuring that data successfully written to Garage is stored on at - least two nodes. - - - `3-dangerous`: a variant of replication mode `3` that lowers both the read - and write quorums to `1`, to allow you to both read and write to your - cluster when several nodes (or nodes in several zones) are unavailable. It - is the least consistent mode of operation proposed by Garage, and also one - that should probably never be used. - -Note that in modes `2` and `3`, -if at least the same number of zones are available, an arbitrary number of failures in -any given zone is tolerated as copies of data will be spread over several zones. - -**Make sure `replication_mode` is the same in the configuration files of all nodes. -Never run a Garage cluster where that is not the case.** - -The quorums associated with each replication mode are described below: - -| `replication_mode` | Number of replicas | Write quorum | Read quorum | Read-after-write consistency? | -| ------------------ | ------------------ | ------------ | ----------- | ----------------------------- | -| `none` or `1` | 1 | 1 | 1 | yes | -| `2` | 2 | 2 | 1 | yes | -| `2-dangerous` | 2 | 1 | 1 | NO | -| `3` | 3 | 2 | 2 | yes | -| `3-degraded` | 3 | 2 | 1 | NO | -| `3-dangerous` | 3 | 1 | 1 | NO | - -Changing the `replication_mode` between modes with the same number of replicas -(e.g. from `3` to `3-degraded`, or from `2-dangerous` to `2`), can be done easily by -just changing the `replication_mode` parameter in your config files and restarting all your -Garage nodes. - -It is also technically possible to change the replication mode to a mode with a -different numbers of replicas, although it's a dangerous operation that is not -officially supported. This requires you to delete the existing cluster layout -and create a new layout from scratch, meaning that a full rebalancing of your -cluster's data will be needed. To do it, shut down your cluster entirely, -delete the `custer_layout` files in the meta directories of all your nodes, -update all your configuration files with the new `replication_mode` parameter, -restart your cluster, and then create a new layout with all the nodes you want -to keep. Rebalancing data will take some time, and data might temporarily -appear unavailable to your users. It is recommended to shut down public access -to the cluster while rebalancing is in progress. In theory, no data should be -lost as rebalancing is a routine operation for Garage, although we cannot -guarantee you that everything will go right in such an extreme scenario. - -### `compression_level` +#### `compression_level` {#compression_level} Zstd compression level to use for storing blocks. @@ -334,7 +588,7 @@ Compression is done synchronously, setting a value too high will add latency to This value can be different between nodes, compression is done by the node which receive the API call. -### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) +#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret} Garage uses a secret key, called an RPC secret, that is shared between all nodes of the cluster in order to identify these nodes and allow them to @@ -346,7 +600,10 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is given in the configuration variable `rpc_secret_file`, or specified as an environment variable `GARAGE_RPC_SECRET`. -### `rpc_bind_addr` +Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file +storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable. + +#### `rpc_bind_addr` {#rpc_bind_addr} The address and port on which to bind for inter-cluster communcations (reffered to as RPC for remote procedure calls). @@ -355,14 +612,33 @@ the node, even in the case of a NAT: the NAT should be configured to forward the port number to the same internal port nubmer. This means that if you have several nodes running behind a NAT, they should each use a different RPC port number. -### `rpc_public_addr` +#### `rpc_bind_outgoing` (since `v0.9.2`) {#rpc_bind_outgoing} + +If enabled, pre-bind all sockets for outgoing connections to the same IP address +used for listening (the IP address specified in `rpc_bind_addr`) before +trying to connect to remote nodes. +This can be necessary if a node has multiple IP addresses, +but only one is allowed or able to reach the other nodes, +for instance due to firewall rules or specific routing configuration. + +Disabled by default. + +#### `rpc_public_addr` {#rpc_public_addr} The address and port that other nodes need to use to contact this node for RPC calls. **This parameter is optional but recommended.** In case you have a NAT that binds the RPC port to a port that is different on your public IP, this field might help making it work. -### `bootstrap_peers` +#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet} +In case `rpc_public_addr` is not set, but autodiscovery is used, this allows +filtering the list of automatically discovered IPs to a specific subnet. + +For example, if nodes should pick *their* IP inside a specific subnet, but you +don't want to explicitly write the IP down (as it's dynamic, or you want to +share configs across nodes), you can use this option. + +#### `bootstrap_peers` {#bootstrap_peers} A list of peer identifiers on which to contact other Garage peers of this cluster. These peer identifiers have the following syntax: @@ -378,43 +654,61 @@ be obtained by running `garage node id` and then included directly in the key will be returned by `garage node id` and you will have to add the IP yourself. +#### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets} -## The `[consul_discovery]` section +Garage checks the permissions of your secret files to make sure they're not +world-readable. In some cases, the check might fail and consider your files as +world-readable even if they're not, for instance when using Posix ACLs. + +Setting `allow_world_readable_secrets` to `true` bypass this +permission verification. + +Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS` +environment variable to `true` to bypass the permissions check. + +#### `allow_punycode` {#allow_punycode} + +Allow creating buckets with names containing punycode. When used for buckets served +as websites, this allows using almost any unicode character in the domain name. + +Default to `false`. + +### The `[consul_discovery]` section Garage supports discovering other nodes of the cluster using Consul. For this to work correctly, nodes need to know their IP address by which they can be reached by other nodes of the cluster, which should be set in `rpc_public_addr`. -### `consul_http_addr` and `service_name` +#### `consul_http_addr` {#consul_http_addr} The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server. -### `api` +#### `api` {#consul_api} Two APIs for service registration are supported: `catalog` and `agent`. `catalog`, the default, will register a service using the `/v1/catalog` endpoints, enabling mTLS if `client_cert` and `client_key` are provided. The `agent` API uses the `v1/agent` endpoints instead, where an optional `token` may be provided. -### `service_name` +#### `service_name` {#consul_service_name} `service_name` should be set to the service name under which Garage's RPC ports are announced. -### `client_cert`, `client_key` +#### `client_cert`, `client_key` {#consul_client_cert_and_key} TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so. Only available when `api = "catalog"`. -### `ca_cert` +#### `ca_cert` {#consul_ca_cert} TLS CA certificate to use when communicating with Consul over TLS. -### `tls_skip_verify` +#### `tls_skip_verify` {#consul_tls_skip_verify} Skip server hostname verification in TLS handshake. `ca_cert` is ignored when this is set. -### `token` +#### `token` {#consul_token} Uses the provided token for communication with Consul. Only available when `api = "agent"`. The policy assigned to this token should at least have these rules: @@ -434,47 +728,49 @@ node_prefix "" { } ``` -### `tags` and `meta` +#### `tags` and `meta` {#consul_tags_and_meta} Additional list of tags and map of service meta to add during service registration. -## The `[kubernetes_discovery]` section +### The `[kubernetes_discovery]` section Garage supports discovering other nodes of the cluster using kubernetes custom resources. For this to work, a `[kubernetes_discovery]` section must be present with at least the `namespace` and `service_name` parameters. -### `namespace` +#### `namespace` {#kube_namespace} `namespace` sets the namespace in which the custom resources are configured. -### `service_name` +#### `service_name` {#kube_service_name} `service_name` is added as a label to the advertised resources to filter them, to allow for multiple deployments in a single namespace. -### `skip_crd` +#### `skip_crd` {#kube_skip_crd} `skip_crd` can be set to true to disable the automatic creation and patching of the `garagenodes.deuxfleurs.fr` CRD. You will need to create the CRD manually. -## The `[s3_api]` section +### The `[s3_api]` section -### `api_bind_addr` +#### `api_bind_addr` {#s3_api_bind_addr} The IP and port on which to bind for accepting S3 API calls. This endpoint does not suport TLS: a reverse proxy should be used to provide it. -### `s3_region` +Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode. + +#### `s3_region` {#s3_region} Garage will accept S3 API calls that are targetted to the S3 region defined here. API calls targetted to other regions will fail with a AuthorizationHeaderMalformed error message that redirects the client to the correct region. -### `root_domain` {#root_domain} +#### `root_domain` {#s3_root_domain} The optional suffix to access bucket using vhost-style in addition to path-style request. Note path-style requests are always enabled, whether or not vhost-style is configured. @@ -486,18 +782,20 @@ using the hostname `my-bucket.s3.garage.eu`. -## The `[s3_web]` section +### The `[s3_web]` section Garage allows to publish content of buckets as websites. This section configures the behaviour of this module. -### `bind_addr` +#### `bind_addr` {#web_bind_addr} The IP and port on which to bind for accepting HTTP requests to buckets configured for website access. This endpoint does not suport TLS: a reverse proxy should be used to provide it. -### `root_domain` +Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode. + +#### `root_domain` {#web_root_domain} The optional suffix appended to bucket names for the corresponding HTTP Host. @@ -505,40 +803,52 @@ For instance, if `root_domain` is `web.garage.eu`, a bucket called `deuxfleurs.f will be accessible either with hostname `deuxfleurs.fr.web.garage.eu` or with hostname `deuxfleurs.fr`. +#### `add_host_to_metrics` {#web_add_host_to_metrics} -## The `[admin]` section +Whether to include the requested domain name (HTTP `Host` header) in the +Prometheus metrics of the web endpoint. This is disabled by default as the +number of possible values is not bounded and can be a source of cardinality +explosion in the exported metrics. + + +### The `[admin]` section Garage has a few administration capabilities, in particular to allow remote monitoring. These features are detailed below. -### `api_bind_addr` +#### `api_bind_addr` {#admin_api_bind_addr} If specified, Garage will bind an HTTP server to this port and address, on which it will listen to requests for administration features. See [administration API reference](@/documentation/reference-manual/admin-api.md) to learn more about these features. -### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) +Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons, +the socket will have 0220 mode. Make sure to set user and group permissions accordingly. + +#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token} The token for accessing the Metrics endpoint. If this token is not set, the Metrics endpoint can be accessed without access control. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `metrics_token` was introduced in Garage `v0.7.2`. `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. -### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) +#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token} The token for accessing all of the other administration endpoints. If this token is not set, access to these endpoints is disabled entirely. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `admin_token` was introduced in Garage `v0.7.2`. `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. -### `trace_sink` +#### `trace_sink` {#admin_trace_sink} Optionally, the address of an OpenTelemetry collector. If specified, Garage will send traces in the OpenTelemetry format to this endpoint. These diff --git a/doc/book/reference-manual/features.md b/doc/book/reference-manual/features.md index 2f8e633a..481aef01 100644 --- a/doc/book/reference-manual/features.md +++ b/doc/book/reference-manual/features.md @@ -37,6 +37,21 @@ A Garage cluster can very easily evolve over time, as storage nodes are added or Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies. Read about cluster layout management [here](@/documentation/operations/layout.md). +### Several replication modes + +Garage supports a variety of replication modes, with configurable replica count, +and with various levels of consistency, in order to adapt to a variety of usage scenarios. +Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_factor) +to select the replication mode best suited to your use case (hint: in most cases, `replication_factor = 3` is what you want). + +### Compression and deduplication + +All data stored in Garage is deduplicated, and optionnally compressed using +Zstd. Objects uploaded to Garage are chunked in blocks of constant sizes (see +[`block_size`](@/documentation/reference-manual/configuration.md#block_size)), +and the hashes of individual blocks are used to dispatch them to storage nodes +and to deduplicate them. + ### No RAFT slowing you down It might seem strange to tout the absence of something as a desirable feature, @@ -46,14 +61,7 @@ directed to a Garage cluster can be handled independently of one another instead of going through a central bottleneck (the leader node). As a consequence, requests can be handled much faster, even in cases where latency between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this). -This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections. - -### Several replication modes - -Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data, -and with various levels of consistency, in order to adapt to a variety of usage scenarios. -Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode) -to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want). +This is particularly useful when nodes are far from one another and talk to one other through standard Internet connections. ### Web server for static websites diff --git a/doc/book/reference-manual/monitoring.md b/doc/book/reference-manual/monitoring.md index 97c533d3..4696791e 100644 --- a/doc/book/reference-manual/monitoring.md +++ b/doc/book/reference-manual/monitoring.md @@ -27,6 +27,112 @@ Exposes the Garage replication factor configured on the node garage_replication_factor 3 ``` +#### `garage_local_disk_avail` and `garage_local_disk_total` (gauge) + +Reports the available and total disk space on each node, for data and metadata separately. + +``` +garage_local_disk_avail{volume="data"} 540341960704 +garage_local_disk_avail{volume="metadata"} 540341960704 +garage_local_disk_total{volume="data"} 763063566336 +garage_local_disk_total{volume="metadata"} 763063566336 +``` + +### Cluster health status metrics + +#### `cluster_healthy` (gauge) + +Whether all storage nodes are connected (0 or 1) + +``` +cluster_healthy 0 +``` + +#### `cluster_available` (gauge) + +Whether all requests can be served, even if some storage nodes are disconnected + +``` +cluster_available 1 +``` + +#### `cluster_connected_nodes` (gauge) + +Number of nodes currently connected + +``` +cluster_connected_nodes 3 +``` + +#### `cluster_known_nodes` (gauge) + +Number of nodes already seen once in the cluster + +``` +cluster_known_nodes 3 +``` + +#### `cluster_layout_node_connected` (gauge) + +Connection status for individual nodes of the cluster layout + +``` +cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +``` + +#### `cluster_layout_node_disconnected_time` (gauge) + +Time (in seconds) since last connection to individual nodes of the cluster layout + +``` +cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +``` + +#### `cluster_storage_nodes` (gauge) + +Number of storage nodes declared in the current layout + +``` +cluster_storage_nodes 4 +``` + +#### `cluster_storage_nodes_ok` (gauge) + +Number of storage nodes currently connected + +``` +cluster_storage_nodes_ok 3 +``` + +#### `cluster_partitions` (gauge) + +Number of partitions in the layout (this is always 256) + +``` +cluster_partitions 256 +``` + +#### `cluster_partitions_all_ok` (gauge) + +Number of partitions for which all storage nodes are connected + +``` +cluster_partitions_all_ok 64 +``` + +#### `cluster_partitions_quorum` (gauge) + +Number of partitions for which we have a quorum of connected nodes and all requests can be served + +``` +cluster_partitions_quorum 256 +``` + ### Metrics of the API endpoints #### `api_admin_request_counter` (counter) @@ -119,6 +225,17 @@ block_bytes_read 120586322022 block_bytes_written 3386618077 ``` +#### `block_ram_buffer_free_kb` (gauge) + +Kibibytes available for buffering blocks that have to be sent to remote nodes. +When clients send too much data to this node and a storage node is not receiving +data fast enough due to slower network conditions, this will decrease down to +zero and backpressure will be applied. + +``` +block_ram_buffer_free_kb 219829 +``` + #### `block_compression_level` (counter) Exposes the block compression level configured for the Garage node. @@ -275,7 +392,7 @@ table_merkle_updater_todo_queue_length{table_name="block_ref"} 0 #### `table_sync_items_received`, `table_sync_items_sent` (counters) -Number of data items sent to/recieved from other nodes during resync procedures +Number of data items sent to/received from other nodes during resync procedures ``` table_sync_items_received{from="",table_name="bucket_v2"} 3 diff --git a/doc/book/reference-manual/s3-compatibility.md b/doc/book/reference-manual/s3-compatibility.md index 1bcfd123..b869b6f4 100644 --- a/doc/book/reference-manual/s3-compatibility.md +++ b/doc/book/reference-manual/s3-compatibility.md @@ -23,16 +23,17 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them. - ## High-level features | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | +| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) | ✅ Implemented | ✅ | ✅ | ❌ | ✅ | | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) | ✅ Implemented | ✅ | ✅ | ❓| ✅ | | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) | +| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ | +| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ | *Note:* OpenIO does not says if it supports presigned URLs. Because it is part of signature v4 and they claim they support it without additional precisions, diff --git a/doc/book/working-documents/design-draft.md b/doc/book/working-documents/design-draft.md index 6560dbed..8d3a31f0 100644 --- a/doc/book/working-documents/design-draft.md +++ b/doc/book/working-documents/design-draft.md @@ -42,7 +42,7 @@ The general principle are similar, but details have not been updated.** A version is defined by the existence of at least one entry in the blocks table for a certain version UUID. We must keep the following invariant: if a version exists in the blocks table, it has to be referenced in the objects table. We explicitly manage concurrent versions of an object: the version timestamp and version UUID columns are index columns, thus we may have several concurrent versions of an object. -Important: before deleting an older version from the objects table, we must make sure that we did a successfull delete of the blocks of that version from the blocks table. +Important: before deleting an older version from the objects table, we must make sure that we did a successful delete of the blocks of that version from the blocks table. Thus, the workflow for reading an object is as follows: @@ -95,7 +95,7 @@ Known issue: if someone is reading from a version that we want to delete and the Usefull metadata: - list of versions that reference this block in the Casandra table, so that we can do GC by checking in Cassandra that the lines still exist -- list of other nodes that we know have acknowledged a write of this block, usefull in the rebalancing algorithm +- list of other nodes that we know have acknowledged a write of this block, useful in the rebalancing algorithm Write strategy: have a single thread that does all write IO so that it is serialized (or have several threads that manage independent parts of the hash space). When writing a blob, write it to a temporary file, close, then rename so that a concurrent read gets a consistent result (either not found or found with whole content). diff --git a/doc/book/working-documents/migration-04.md b/doc/book/working-documents/migration-04.md index d9d3ede1..52c56737 100644 --- a/doc/book/working-documents/migration-04.md +++ b/doc/book/working-documents/migration-04.md @@ -68,7 +68,7 @@ The migration steps are as follows: 5. Turn off Garage 0.3 6. Backup metadata folders if you can (i.e. if you have space to do it - somewhere). Backuping data folders could also be usefull but that's much + somewhere). Backuping data folders could also be useful but that's much harder to do. If your filesystem supports snapshots, this could be a good time to use them. diff --git a/doc/book/working-documents/migration-09.md b/doc/book/working-documents/migration-09.md new file mode 100644 index 00000000..ba758093 --- /dev/null +++ b/doc/book/working-documents/migration-09.md @@ -0,0 +1,72 @@ ++++ +title = "Migrating from 0.8 to 0.9" +weight = 12 ++++ + +**This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster. +We don't recommend trying to migrate to 0.9 directly from 0.7 or older.** + +This migration procedure has been tested on several clusters without issues. +However, it is still a *critical procedure* that might cause issues. +**Make sure to back up all your data before attempting it!** + +You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md). + +The following are **breaking changes** in Garage v0.9 that require your attention when migrating: + +- LMDB is now the default metadata db engine and Sled is deprecated. If you were using Sled, make sure to specify `db_engine = "sled"` in your configuration file, or take the time to [convert your database](https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0). + +- Capacity values are now in actual byte units. The translation from the old layout will assign 1 capacity = 1Gb by default, which might be wrong for your cluster. This does not cause any data to be moved around, but you might want to re-assign correct capacity values post-migration. + +- Multipart uploads that were started in Garage v0.8 will not be visible in Garage v0.9 and will have to be restarted from scratch. + +- Changes to the admin API: some `v0/` endpoints have been replaced by `v1/` counterparts with updated/uniformized syntax. All other endpoints have also moved to `v1/` by default, without syntax changes, but are still available under `v0/` for compatibility. + + +## Simple migration procedure (takes cluster offline for a while) + +The migration steps are as follows: + +1. Disable API and web access. You may do this by stopping your reverse proxy or by commenting out + the `api_bind_addr` values in your `config.toml` file and restarting Garage. +2. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`, + check the logs and check that all data seems to be synced correctly between + nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.) +3. Check that the block resync queue and Merkle queue are empty: + run `garage stats -a` to query them or inspect metrics in the Grafana dashboard. +4. Turn off Garage v0.8 +5. **Backup the metadata folder of all your nodes!** For instance, use the following command + if your metadata directory is `/var/lib/garage/meta`: `cd /var/lib/garage ; tar -acf meta-v0.8.tar.zst meta/` +6. Install Garage v0.9 +7. Update your configuration file if necessary. +8. Turn on Garage v0.9 +9. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`. + Wait for a full table sync to run. +10. Your upgraded cluster should be in a working state. Re-enable API and Web + access and check that everything went well. +11. Monitor your cluster in the next hours to see if it works well under your production load, report any issue. +12. You might want to assign correct capacity values to all your nodes. Doing so might cause data to be moved + in your cluster, which should also be monitored carefully. + +## Minimal downtime migration procedure + +The migration to Garage v0.9 can be done with almost no downtime, +by restarting all nodes at once in the new version. + +The migration steps are as follows: + +1. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`, + check the logs and check that all data seems to be synced correctly between + nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.) + +2. Turn off each node individually; back up its metadata folder (see above); turn it back on again. + This will allow you to take a backup of all nodes without impacting global cluster availability. + You can do all nodes of a single zone at once as this does not impact the availability of Garage. + +3. Prepare your binaries and configuration files for Garage v0.9 + +4. Shut down all v0.8 nodes simultaneously, and restart them all simultaneously in v0.9. + Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to achieve this as fast as possible. + Garage v0.9 should be in a working state as soon as it starts. + +5. Proceed with repair and monitoring as described in steps 9-12 above. diff --git a/doc/book/working-documents/migration-1.md b/doc/book/working-documents/migration-1.md new file mode 100644 index 00000000..b6c0bb85 --- /dev/null +++ b/doc/book/working-documents/migration-1.md @@ -0,0 +1,77 @@ ++++ +title = "Migrating from 0.9 to 1.0" +weight = 11 ++++ + +**This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster. +We don't recommend trying to migrate to 1.0 directly from 0.8 or older.** + +This migration procedure has been tested on several clusters without issues. +However, it is still a *critical procedure* that might cause issues. +**Make sure to back up all your data before attempting it!** + +You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md). + +## Changes introduced in v1.0 + +The following are **breaking changes** in Garage v1.0 that require your attention when migrating: + +- The Sled metadata db engine has been **removed**. If your cluster was still + using Sled, you will need to **use a Garage v0.9.x binary** to convert the + database using the `garage convert-db` subcommand. See + [here](@/documentation/reference-manual/configuration.md#db_engine) for the + details of the procedure. + +The following syntax changes have been made to the configuration file: + +- The `replication_mode` parameter has been split into two parameters: + [`replication_factor`](@/documentation/reference-manual/configuration.md#replication_factor) + and + [`consistency_mode`](@/documentation/reference-manual/configuration.md#consistency_mode). + The old syntax using `replication_mode` is still supported for legacy + reasons and can still be used. + +- The parameters `sled_cache_capacity` and `sled_flush_every_ms` have been removed. + +## Migration procedure + +The migration to Garage v1.0 can be done with almost no downtime, +by restarting all nodes at once in the new version. + +The migration steps are as follows: + +1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that + all data seems to be synced correctly between nodes. If you have time, do + additional `garage repair` procedures (`blocks`, `versions`, `block_refs`, + etc.) + +2. Ensure you have a snapshot of your Garage installation that you can restore + to in case the upgrade goes wrong: + + - If you are running Garage v0.9.4 or later, use the `garage meta snapshot + --all` to make a backup snapshot of the metadata directories of your nodes + for backup purposes, and save a copy of the following files in the + metadata directories of your nodes: `cluster_layout`, `data_layout`, + `node_key`, `node_key.pub`. + + - If you are running a filesystem such as ZFS or BTRFS that support + snapshotting, you can create a filesystem-level snapshot to be used as a + restoration point if needed. + + - In other cases, make a backup using the old procedure: turn off each node + individually; back up its metadata folder (for instance, use the following + command if your metadata directory is `/var/lib/garage/meta`: `cd + /var/lib/garage ; tar -acf meta-v0.9.tar.zst meta/`); turn it back on + again. This will allow you to take a backup of all nodes without + impacting global cluster availability. You can do all nodes of a single + zone at once as this does not impact the availability of Garage. + +3. Prepare your updated binaries and configuration files for Garage v1.0 + +4. Shut down all v0.9 nodes simultaneously, and restart them all simultaneously + in v1.0. Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to + achieve this as fast as possible. Garage v1.0 should be in a working state + as soon as enough nodes have started. + +5. Monitor your cluster in the following hours to see if it works well under + your production load. diff --git a/doc/book/working-documents/testing-strategy.md b/doc/book/working-documents/testing-strategy.md index 7d6be8ef..fff706d7 100644 --- a/doc/book/working-documents/testing-strategy.md +++ b/doc/book/working-documents/testing-strategy.md @@ -37,7 +37,7 @@ There are two reasons for this: Reminder: rules of simplicity, concerning changes to Garage's source code. Always question what we are doing. -Never do anything just because it looks nice or because we "think" it might be usefull at some later point but without knowing precisely why/when. +Never do anything just because it looks nice or because we "think" it might be useful at some later point but without knowing precisely why/when. Only do things that make perfect sense in the context of what we currently know. ## References diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 411f6418..3ee948cb 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -8,9 +8,9 @@ listen address is specified in the `[admin]` section of the configuration file (see [configuration file reference](@/documentation/reference-manual/configuration.md)) -**WARNING.** At this point, there is no comittement to stability of the APIs described in this document. -We will bump the version numbers prefixed to each API endpoint at each time the syntax -or semantics change, meaning that code that relies on these endpoint will break +**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document. +We will bump the version numbers prefixed to each API endpoint each time the syntax +or semantics change, meaning that code that relies on these endpoints will break when changes are introduced. The Garage administration API was introduced in version 0.7.2, this document @@ -19,7 +19,7 @@ does not apply to older versions of Garage. ## Access control -The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: +The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without @@ -69,11 +69,10 @@ Example response body: ```json { - "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", - "garageVersion": "git:v0.9.0-dev", + "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "garageVersion": "v1.3.0", "garageFeatures": [ "k2v", - "sled", "lmdb", "sqlite", "metrics", @@ -81,83 +80,92 @@ Example response body: ], "rustVersion": "1.68.0", "dbEngine": "LMDB (using Heed crate)", - "knownNodes": [ + "layoutVersion": 5, + "nodes": [ { - "id": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", - "addr": "10.0.0.11:3901", + "id": "62b218d848e86a64f7fe1909735f29a4350547b54c4b204f91246a14eb0a1a8c", + "role": { + "id": "62b218d848e86a64f7fe1909735f29a4350547b54c4b204f91246a14eb0a1a8c", + "zone": "dc1", + "capacity": 100000000000, + "tags": [] + }, + "addr": "10.0.0.3:3901", + "hostname": "node3", "isUp": true, - "lastSeenSecsAgo": 9, - "hostname": "node1" + "lastSeenSecsAgo": 12, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } }, { - "id": "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff", - "addr": "10.0.0.12:3901", + "id": "a11c7cf18af297379eff8688360155fe68d9061654449ba0ce239252f5a7487f", + "role": null, + "addr": "10.0.0.2:3901", + "hostname": "node2", "isUp": true, - "lastSeenSecsAgo": 1, - "hostname": "node2" + "lastSeenSecsAgo": 11, + "draining": true, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } }, { - "id": "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27", - "addr": "10.0.0.21:3901", + "id": "a235ac7695e0c54d7b403943025f57504d500fdcc5c3e42c71c5212faca040a2", + "role": { + "id": "a235ac7695e0c54d7b403943025f57504d500fdcc5c3e42c71c5212faca040a2", + "zone": "dc1", + "capacity": 100000000000, + "tags": [] + }, + "addr": "127.0.0.1:3904", + "hostname": "lindy", "isUp": true, - "lastSeenSecsAgo": 7, - "hostname": "node3" + "lastSeenSecsAgo": 2, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } }, { - "id": "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b", - "addr": "10.0.0.22:3901", + "id": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "role": { + "id": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "zone": "dc1", + "capacity": 100000000000, + "tags": [] + }, + "addr": "10.0.0.1:3901", + "hostname": "node1", "isUp": true, - "lastSeenSecsAgo": 1, - "hostname": "node4" + "lastSeenSecsAgo": 3, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } } - ], - "layout": { - "version": 12, - "roles": [ - { - "id": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", - "zone": "dc1", - "capacity": 10737418240, - "tags": [ - "node1" - ] - }, - { - "id": "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff", - "zone": "dc1", - "capacity": 10737418240, - "tags": [ - "node2" - ] - }, - { - "id": "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27", - "zone": "dc2", - "capacity": 10737418240, - "tags": [ - "node3" - ] - } - ], - "stagedRoleChanges": [ - { - "id": "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b", - "remove": false, - "zone": "dc2", - "capacity": 10737418240, - "tags": [ - "node4" - ] - } - { - "id": "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27", - "remove": true, - "zone": null, - "capacity": null, - "tags": null, - } - ] - } + ] } ``` diff --git a/doc/drafts/k2v-spec.md b/doc/drafts/k2v-spec.md index faa1a247..f9696717 100644 --- a/doc/drafts/k2v-spec.md +++ b/doc/drafts/k2v-spec.md @@ -146,7 +146,7 @@ in a bucket, as the partition key becomes the sort key in the index. How indexing works: - Each node keeps a local count of how many items it stores for each partition, - in a local Sled tree that is updated atomically when an item is modified. + in a local database tree that is updated atomically when an item is modified. - These local counters are asynchronously stored in the index table which is a regular Garage table spread in the network. Counters are stored as LWW values, so basically the final table will have the following structure: @@ -562,7 +562,7 @@ token>", v: ["", ...] }`, with the following fields: - in case of concurrent update and deletion, a `null` is added to the list of concurrent values - if the `tombstones` query parameter is set to `true`, tombstones are returned - for items that have been deleted (this can be usefull for inserting after an + for items that have been deleted (this can be useful for inserting after an item that has been deleted, so that the insert is not considered concurrent with the delete). Tombstones are returned as tuples in the same format with only `null` values diff --git a/doc/sticker/Garage.pdf b/doc/sticker/Garage.pdf new file mode 100644 index 00000000..1bfb4a16 Binary files /dev/null and b/doc/sticker/Garage.pdf differ diff --git a/doc/sticker/Garage.png b/doc/sticker/Garage.png new file mode 100644 index 00000000..1bfd61b1 Binary files /dev/null and b/doc/sticker/Garage.png differ diff --git a/doc/sticker/Garage_NGI.pdf b/doc/sticker/Garage_NGI.pdf index 1359a30d..43f511e7 100644 Binary files a/doc/sticker/Garage_NGI.pdf and b/doc/sticker/Garage_NGI.pdf differ diff --git a/doc/talks/.gitignore b/doc/talks/.gitignore new file mode 100644 index 00000000..9b421064 --- /dev/null +++ b/doc/talks/.gitignore @@ -0,0 +1 @@ +.direnv/ diff --git a/doc/talks/2023-09-20-ocp/.gitignore b/doc/talks/2023-09-20-ocp/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2023-09-20-ocp/Makefile b/doc/talks/2023-09-20-ocp/Makefile new file mode 100644 index 00000000..554f7b97 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/Makefile @@ -0,0 +1,34 @@ +ASSETS=assets/consistent_hashing_1.pdf \ + assets/consistent_hashing_2.pdf \ + assets/consistent_hashing_3.pdf \ + assets/consistent_hashing_4.pdf \ + assets/garage_tables.pdf \ + assets/consensus.pdf_tex \ + assets/lattice1.pdf_tex \ + assets/lattice2.pdf_tex \ + assets/lattice3.pdf_tex \ + assets/lattice4.pdf_tex \ + assets/lattice5.pdf_tex \ + assets/lattice6.pdf_tex \ + assets/lattice7.pdf_tex \ + assets/lattice8.pdf_tex \ + assets/latticeB_1.pdf_tex \ + assets/latticeB_2.pdf_tex \ + assets/latticeB_3.pdf_tex \ + assets/latticeB_4.pdf_tex \ + assets/latticeB_5.pdf_tex \ + assets/latticeB_6.pdf_tex \ + assets/latticeB_7.pdf_tex \ + assets/latticeB_8.pdf_tex \ + assets/latticeB_9.pdf_tex \ + assets/latticeB_10.pdf_tex \ + assets/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +assets/%.pdf: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +assets/%.pdf_tex: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2023-09-20-ocp/abstract.md b/doc/talks/2023-09-20-ocp/abstract.md new file mode 100644 index 00000000..b2658868 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/abstract.md @@ -0,0 +1,39 @@ +### (fr) Garage, un système de stockage de données géo-distribué léger et robuste + +Garage est un système de stockage de données léger, géo-distribué, qui +implémente le protocole de stockage S3 de Amazon. Garage est destiné +principalement à l'auto-hébergement sur du matériel courant d'occasion. À ce +titre, il doit tolérer un grand nombre de pannes: coupures de courant, coupures +de connexion Internet, pannes de machines, ... Il doit également être facile à +déployer et à maintenir, afin de pouvoir être facilement utilisé par des +amateurs ou des petites organisations. + +Cette présentation vous proposera un aperçu de Garage et du choix technique +principal qui rend un système comme Garage possible: le refus d'utiliser des +algorithmes de consensus, remplacés avantageusement par des méthodes à +cohérence faible. Notre modèle est fortement inspiré de la base de donnée +Dynamo (DeCandia et al, 2007), et fait usage des types de données CRDT (Shapiro +et al, 2011). Nous exploreront comment ces méthodes s'appliquent à la +construction de l'abstraction "stockage objet" dans un système distribué, et +quelles autres abstractions peuvent ou ne peuvent pas être construites dans ce +modèle. + +### (en) Garage, a lightweight and robust geo-distributed data storage system + +Garage is a lightweight geo-distributed data store that implements the Amazon +S3 object storage protocol. Garage is meant primarily for self-hosting at home +on second-hand commodity hardware, meaning it has to tolerate a wide variety of +failure scenarios such as power cuts, Internet disconnections and machine +crashes. It also has to be easy to deploy and maintain, so that hobbyists and +small organizations can use it without trouble. + +This talk will present Garage and the key technical choice that made Garage +possible: refusing to use consensus algorithms and using instead weak +consistency methods, with a model that is loosely based on that of the Dynamo +database (DeCandia et al, 2007) and that makes heavy use of conflict-free +replicated data types (Shapiro et al, 2011). We will explore how these methods +are suited to building the "object store" abstraction in a distributed system, +and what other abstractions are possible or impossible to build in this model. + + + diff --git a/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png b/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png new file mode 100644 index 00000000..445284a3 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme.png b/doc/talks/2023-09-20-ocp/assets/aerogramme.png new file mode 100644 index 00000000..3aabe3ad Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme.svg b/doc/talks/2023-09-20-ocp/assets/aerogramme.svg new file mode 100644 index 00000000..0c1ee127 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/aerogramme.svg @@ -0,0 +1,1241 @@ + + + + + + K2V APIS3 APIAerogramme + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +IMAPIMAPIMAPMessageindexMessagebodies diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf new file mode 100644 index 00000000..71a90f26 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png new file mode 100644 index 00000000..fb81b460 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf new file mode 100644 index 00000000..87e42eed Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png new file mode 100644 index 00000000..f9e2df14 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf new file mode 100644 index 00000000..0606e059 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png new file mode 100644 index 00000000..c3b015a1 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf new file mode 100644 index 00000000..8fea81c7 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png new file mode 100644 index 00000000..ed2077d9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/alex.jpg b/doc/talks/2023-09-20-ocp/assets/alex.jpg new file mode 100644 index 00000000..eac0f0a9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/alex.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/atuin.jpg b/doc/talks/2023-09-20-ocp/assets/atuin.jpg new file mode 100644 index 00000000..f2fbd61d Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/atuin.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/compatibility.png b/doc/talks/2023-09-20-ocp/assets/compatibility.png new file mode 100644 index 00000000..ce364a9b Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/compatibility.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/consensus.svg b/doc/talks/2023-09-20-ocp/assets/consensus.svg new file mode 100644 index 00000000..8321e383 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consensus.svg @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + $\bot$ + + + + $x$ + + + $propose(x) / x$ + $propose(y) / x$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg new file mode 100644 index 00000000..f8d24fd8 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg @@ -0,0 +1,301 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg new file mode 100644 index 00000000..5ac8faf6 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg @@ -0,0 +1,334 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg new file mode 100644 index 00000000..fdfd3efc --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg @@ -0,0 +1,358 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg new file mode 100644 index 00000000..95ed0e02 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg b/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg new file mode 100644 index 00000000..c298c22b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + D + F + diff --git a/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png b/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png new file mode 100644 index 00000000..7c7411cd Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf new file mode 100644 index 00000000..a54a163c Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage.drawio.png new file mode 100644 index 00000000..386dd862 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png new file mode 100644 index 00000000..8562fbcf Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..422c9343 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..05a9710e Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf new file mode 100644 index 00000000..a94b3572 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png new file mode 100644 index 00000000..2e7b5af0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_tables.svg b/doc/talks/2023-09-20-ocp/assets/garage_tables.svg new file mode 100644 index 00000000..c7172713 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/garage_tables.svg @@ -0,0 +1,537 @@ + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + Object + + + + + + bucket + + + + + + file path + + = partition key + + = sort key + + + + + + Version 1 + deleted + + + + + + + Version 2 + id + + size + MIME type + ... + + + + + + Version + + id + h(block 1) + h(block 2) + ... + + + + + Data block + + hash + data + + + + Objects table + Versions table + Blocks table + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice1.svg b/doc/talks/2023-09-20-ocp/assets/lattice1.svg new file mode 100644 index 00000000..8bfa5aa7 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice1.svg @@ -0,0 +1,433 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice2.svg b/doc/talks/2023-09-20-ocp/assets/lattice2.svg new file mode 100644 index 00000000..adcd92cb --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice2.svg @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice3.svg b/doc/talks/2023-09-20-ocp/assets/lattice3.svg new file mode 100644 index 00000000..640dc468 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice3.svg @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice4.svg b/doc/talks/2023-09-20-ocp/assets/lattice4.svg new file mode 100644 index 00000000..b2a99e28 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice4.svg @@ -0,0 +1,525 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice5.svg b/doc/talks/2023-09-20-ocp/assets/lattice5.svg new file mode 100644 index 00000000..bc6b7195 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice5.svg @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice6.svg b/doc/talks/2023-09-20-ocp/assets/lattice6.svg new file mode 100644 index 00000000..176b1715 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice6.svg @@ -0,0 +1,553 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice7.svg b/doc/talks/2023-09-20-ocp/assets/lattice7.svg new file mode 100644 index 00000000..7ce8bda8 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice7.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice8.svg b/doc/talks/2023-09-20-ocp/assets/lattice8.svg new file mode 100644 index 00000000..c94a69b2 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice8.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg new file mode 100644 index 00000000..92232a1b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg new file mode 100644 index 00000000..34c24e0d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg @@ -0,0 +1,715 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + return $\{b\}$ + + $\to \{b\}$ + ${\Large\textbf{??!}}$~~~~~$\{a\} \not\sqsubseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg new file mode 100644 index 00000000..c07cba2b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg new file mode 100644 index 00000000..198d1f5d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg new file mode 100644 index 00000000..c5f6148d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg @@ -0,0 +1,587 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg new file mode 100644 index 00000000..c2b668be --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg @@ -0,0 +1,604 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg new file mode 100644 index 00000000..980823fc --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg @@ -0,0 +1,632 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg new file mode 100644 index 00000000..154c0b7d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg @@ -0,0 +1,654 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + $read()$: + ; + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg new file mode 100644 index 00000000..21766415 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg @@ -0,0 +1,671 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg new file mode 100644 index 00000000..b60f8afe --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg @@ -0,0 +1,699 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + return $\{b\}$ + + $\to \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/location-aware.png b/doc/talks/2023-09-20-ocp/assets/location-aware.png new file mode 100644 index 00000000..f5966865 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/location-aware.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/logo_chatons.png b/doc/talks/2023-09-20-ocp/assets/logo_chatons.png new file mode 100644 index 00000000..890cf17e Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/logo_chatons.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/map.png b/doc/talks/2023-09-20-ocp/assets/map.png new file mode 100644 index 00000000..1dff3ab6 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/map.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/minio.png b/doc/talks/2023-09-20-ocp/assets/minio.png new file mode 100644 index 00000000..a71e9ccc Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/minio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/neptune.jpg b/doc/talks/2023-09-20-ocp/assets/neptune.jpg new file mode 100644 index 00000000..61fcbff6 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/neptune.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/rust_logo.png b/doc/talks/2023-09-20-ocp/assets/rust_logo.png new file mode 100644 index 00000000..0e4809ec Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/rust_logo.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png new file mode 100644 index 00000000..c92f0774 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png new file mode 100644 index 00000000..849b8336 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png new file mode 100644 index 00000000..86dcce89 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png new file mode 100644 index 00000000..e7d4e288 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png new file mode 100644 index 00000000..472312a0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide1.png b/doc/talks/2023-09-20-ocp/assets/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide2.png b/doc/talks/2023-09-20-ocp/assets/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide3.png b/doc/talks/2023-09-20-ocp/assets/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide3.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB1.png b/doc/talks/2023-09-20-ocp/assets/slideB1.png new file mode 100644 index 00000000..b14b6070 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB2.png b/doc/talks/2023-09-20-ocp/assets/slideB2.png new file mode 100644 index 00000000..a881a796 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB3.png b/doc/talks/2023-09-20-ocp/assets/slideB3.png new file mode 100644 index 00000000..830709d2 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB3.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slides.svg b/doc/talks/2023-09-20-ocp/assets/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/slidesB.svg b/doc/talks/2023-09-20-ocp/assets/slidesB.svg new file mode 100644 index 00000000..c0a6e97c --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database diff --git a/doc/talks/2023-09-20-ocp/talk.pdf b/doc/talks/2023-09-20-ocp/talk.pdf new file mode 100644 index 00000000..f7b96ef9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/talk.pdf differ diff --git a/doc/talks/2023-09-20-ocp/talk.tex b/doc/talks/2023-09-20-ocp/talk.tex new file mode 100644 index 00000000..21ce54b0 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/talk.tex @@ -0,0 +1,1008 @@ +\nonstopmode +\documentclass[aspectratio=169]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\subtitle{a lightweight and robust geo-distributed data storage system} +\author{Alex Auvolat, Deuxfleurs} +\date{OCamlPro, 2023-09-20} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{assets/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{assets/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize (we want good uptime/availability with low supervision)} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make a \underline{stable} system} + + Enterprise-grade systems typically employ: + \vspace{1em} + \begin{itemize} + \item RAID + \item Redundant power grid + UPS + \item Redundant Internet connections + \item Low-latency links + \item ... + \end{itemize} + \vspace{1em} + $\to$ it's costly and only worth it at DC scale +\end{frame} + +\begin{frame} + \frametitle{How to make a \underline{resilient} system} + + \only<1,4-5>{ + Instead, we use: + \vspace{1em} + \begin{itemize} + \item \textcolor<2->{gray}{Commodity hardware (e.g. old desktop PCs)} + \vspace{.5em} + \item<4-> \textcolor<5->{gray}{Commodity Internet (e.g. FTTB, FTTH) and power grid} + \vspace{.5em} + \item<5-> \textcolor<6->{gray}{\textbf{Geographical redundancy} (multi-site replication)} + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/neptune.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/atuin.jpg} + \end{center} + } + \only<6>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{assets/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{assets/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec (e.g.~locks) + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[height=6em]{../2020-12-02_wide-team/img/Amazon-S3.jpg} + \hspace{3em} + \includegraphics[height=5em]{assets/minio.png} + \hspace{3em} + \includegraphics[height=6em]{../../logo/garage_hires_crop.png} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + + MinIO is self-hostable but not suited for geo-distributed deployments + + \vspace{1em} + + \textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store} +\end{frame} + + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a key-value store: + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|p{8cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + +\end{frame} + + +\begin{frame} + \frametitle{Two big problems} + \begin{enumerate} + \item \textbf{How to place data on different nodes?}\\ + \vspace{1em} + \underline{Constraints:} heterogeneous hardware\\ + \underline{Objective:} $n$ copies of everything, maximize usable capacity, maximize resilience\\ + \vspace{1em} + $\to$ the Dynamo model + optimization algorithms + \vspace{2em} + \item<2-> \textbf{How to guarantee consistency?}\\ + \vspace{1em} + \underline{Constraints:} slow network (geographical distance), node unavailability/crashes\\ + \underline{Objective:} maximize availability, read-after-write guarantee\\ + \vspace{1em} + $\to$ CRDTs, monotonicity, read and write quorums + \end{enumerate} +\end{frame} + +\section{Problem 1: placing data} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \textbf{Two keys:} + \begin{itemize} + \item Partition key: used to divide data into partitions {\small (a.k.a.~shards)} + \item Sort key: used to identify items inside a partition + \end{itemize} + + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|l|p{3cm}|} + \hline + \textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & \texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \begin{itemize} + \item Data with different partition keys is stored independently,\\ + on a different set of nodes\\ + \vspace{.5em} + $\to$ no easy way to list all partition keys\\ + $\to$ no cross-shard transactions\\ + \vspace{2em} + \item Placing data: hash the partition key, select nodes accordingly\\ + \vspace{.5em} + $\to$ distributed hash table (DHT) + \vspace{2em} + \item For a given value of the partition key, items can be listed using their sort keys + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How to spread files over different cluster nodes?} + \textbf{Consistent hashing (Dynamo):} + \vspace{1em} + + \begin{center} + \only<1>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_1.pdf}}% + \only<2>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_2.pdf}}% + \only<3>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_3.pdf}}% + \only<4>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_4.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Constraint: location-awareness} + \begin{center} + \includegraphics[width=\linewidth]{assets/location-aware.png} + \end{center} + \vspace{2em} + Garage replicates data on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{Constraint: location-awareness} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/map.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Issues with consistent hashing} + \begin{itemize} + \item Consistent hashing doesn't dispatch data based on geographical location of nodes + \vspace{1em} + \item<2-> Geographically aware adaptation, try 1:\\ + data quantities not well balanced between nodes + \vspace{1em} + \item<3-> Geographically aware adaptation, try 2:\\ + too many reshuffles when adding/removing nodes + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How to spread files over different cluster nodes?} + \textbf{Garage's method: build an index table} + \vspace{1em} + + Realization: we can actually precompute an optimal solution + \vspace{1em} + + \visible<2->{ + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & Io (jupiter) & Drosera (atuin) & Courgette (neptune) \\ + \hline + Partition 1 & Datura (atuin) & Courgette (neptune) & Io (jupiter) \\ + \hline + Partition 2 & Io(jupiter) & Celeri (neptune) & Drosera (atuin) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & Io (jupiter) & Drosera (atuin) \\ + \hline + \end{tabular} + \end{center} + } + \vspace{1em} + \visible<3->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } +\end{frame} + +\begin{frame} + \frametitle{The relationship between \emph{partition} and \emph{partition key}} + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} + \vspace{1em} + \textbf{To read or write an item:} hash partition key + \\ \hspace{5cm} $\to$ determine partition number (first 8 bits) + \\ \hspace{5cm} $\to$ find associated nodes +\end{frame} + +\begin{frame} + \frametitle{Garage's internal data structures} + \centering + \includegraphics[width=.75\columnwidth]{assets/garage_tables.pdf} +\end{frame} + +\begin{frame} + \frametitle{Storing and retrieving files} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{assets/garage2a.drawio.pdf}}% + \only<2>{\includegraphics[width=.45\linewidth]{assets/garage2b.drawio.pdf}}% + \end{center} +\end{frame} + +\section{Problem 2: ensuring consistency} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Consensus-based systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Leader-based:} a leader is elected to coordinate + all reads and writes + \vspace{1em} + \item \textbf{Linearizability} of all operations\\ + (strongest consistency guarantee) + \vspace{1em} + \item Any sequential specification can be implemented as a \textbf{replicated state machine} + \vspace{1em} + \item \textbf{Costly}, the leader is a bottleneck; + leader elections on failure take time + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}{7cm} \visible<2->{ + \textbf{Weakly consistent systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Nodes are equivalent}, any node + can originate a read or write operation + \vspace{1em} + \item \textbf{Read-after-write consistency} with quorums, + eventual consistency without + \vspace{1em} + \item \textbf{Operations have to commute}, i.e.~we + can only implement CRDTs + \vspace{1em} + \item \textbf{Fast}, no single bottleneck;\\ + works the same with offline nodes + \end{itemize} + } \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + \begin{center} + \textbf{From a theoretical point of view:}\\ + + \end{center} + \vspace{2em} + + \hspace{1em} + \begin{minipage}{6.5cm} + \underline{Consensus-based systems:} + + \vspace{1em} + + Require \textbf{additional assumptions} such as a fault detector or a strong RNG\\ + (FLP impossibility theorem) + \end{minipage} + \hfill + \begin{minipage}{6.5cm} + \underline{Weakly consistent systems:} + + \vspace{1em} + + Can be implemented in \textbf{any\\asynchronous message passing\\distributed system} with node crashes + \end{minipage} + \hspace{1em} + + \vspace{3em} + \begin{center} + They represent \textbf{different classes of computational capability}\\ + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + \begin{center} + \textbf{The same objects cannot be implemented in both models.} + \end{center} + \vspace{2em} + + \hspace{1em} + \begin{minipage}{6.5cm} + \underline{Consensus-based systems:} + + \vspace{1em} + + \textbf{Any sequential specification}\\~ + + \vspace{1em} + \textbf{Easier to program for}: just write your program as if it were sequential on a single machine + + \end{minipage} + \hfill + \begin{minipage}{6.5cm} + \underline{Weakly consistent systems:} + + \vspace{1em} + + \textbf{Only CRDTs}\\(conflict-free replicated data types) + + \vspace{1em} + Part of the complexity is \textbf{reported to the consumer of the API}\\~ + \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Understanding the power of consensus} + \textbf{Consensus:} an API with a single operation, $propose(x)$ + \begin{enumerate} + \item nodes all call $propose(x)$ with their proposed value; + \item nodes all receive the same value as a return value, which is one of the proposed values + \end{enumerate} + \vspace{1em} + + \visible<2->{ + \textbf{Equivalent to} a distributed algorithm that gives a total order on all requests + } + \vspace{1em} + + \visible<3->{ + \textbf{Implemented by} this simple replicated state machine: + \vspace{.5em} + \begin{figure} + \centering + \def\svgwidth{.5\textwidth} + \large + \import{assets/}{consensus.pdf_tex} + \end{figure} + \vspace{1em} + } +\end{frame} + +\begin{frame} + \frametitle{Can my object be implemented without consensus?} + \underline{Given the specification of an API:} + \vspace{2em} + \begin{itemize} + \item \textbf{Using this API, we can implement the consensus object} (the $propose$ function)\\ + $\to$ the API is equivalent to consensus/total ordering of messages\\ + $\to$ the API cannot be implemented in a weakly consistent system + \vspace{2em} + \item<2-> \textbf{This API can be implemented using only weak primitives}\\ + (e.g. in the asynchronous message passing model with no further assumption)\\ + $\to$ the API is strictly weaker than consensus\\ + $\to$ we can implement it in Garage! + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Why avoid consensus?} + Consensus can be implemented reasonably well in practice, so why avoid it? + \vspace{2em} + \begin{itemize} + \item \textbf{Software complexity:} RAFT and PAXOS are complex beasts;\\ + harder to prove, harder to reason about + \vspace{1.5em} + \item \textbf{Performance issues:} + \vspace{1em} + \begin{itemize} + \item Theoretical requirements (RNG, failure detector) translate into \textbf{practical costs} + \vspace{1em} + \item The leader is a \textbf{bottleneck} for all requests;\\ + even in leaderless approaches, \textbf{all nodes must process all operations in order} + \vspace{1em} + \item Particularly \textbf{sensitive to higher latency} between nodes + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/endpoint-latency-dc.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{What can we implement without consensus?} + \begin{itemize} + \item Any \textbf{conflict-free replicated data type} (CRDT) + \vspace{1em} + \item<2-> Non-transactional key-value stores such as S3 are equivalent to a simple CRDT:\\ + a map of \textbf{last-writer-wins registers} (each key is its own CRDT) + \vspace{1em} + \item<3-> \textbf{Read-after-write consistency} can be implemented + using quorums on read and write operations + \vspace{1em} + \item<4-> \textbf{Monotonicity of reads} can be implemented with repair-on-read\\ + (makes reads more costly, not implemented in Garage) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: read-after-write consistency} + \begin{figure} + \centering + \def\svgwidth{.8\textwidth} + \only<1>{\import{assets/}{lattice1.pdf_tex}}% + \only<2>{\import{assets/}{lattice2.pdf_tex}}% + \only<3>{\import{assets/}{lattice3.pdf_tex}}% + \only<4>{\import{assets/}{lattice4.pdf_tex}}% + \only<5>{\import{assets/}{lattice5.pdf_tex}}% + \only<6>{\import{assets/}{lattice6.pdf_tex}}% + \only<7>{\import{assets/}{lattice7.pdf_tex}}% + \only<8>{\import{assets/}{lattice8.pdf_tex}}% + \end{figure} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: read-after-write consistency} + \textbf{Property:} If node $A$ did an operation $write(x)$ and received an OK response,\\ + \hspace{2cm} and node $B$ starts an operation $read()$ after $A$ received OK,\\ + \hspace{2cm} then $B$ will read a value $x' \sqsupseteq x$. + + \vspace{1em} + + \hspace{1em} + \begin{minipage}{6.8cm} + \textbf{Algorithm $write(x)$:} + \begin{enumerate} + \item Broadcast $write(x)$ to all nodes + \item Wait for $k > n/2$ nodes to reply OK + \item Return OK + \end{enumerate} + \end{minipage} + \hfill + \begin{minipage}{6.8cm} + \vspace{1em} + \textbf{Algorithm $read()$:} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply\\ + with values $x_1, \dots, x_k$ + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + \end{minipage} + \hspace{1em} + + \vspace{2em} + \textbf{Why does it work?} There is at least one node at the intersection between the two sets of nodes that replied to each request, that ``saw'' $x$ before the $read()$ started ($x_i \sqsupseteq x$). +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: monotonic-reads consistency} + \begin{figure} + \centering + \def\svgwidth{.8\textwidth} + \only<1>{\import{assets/}{latticeB_1.pdf_tex}}% + \only<2>{\import{assets/}{latticeB_2.pdf_tex}}% + \only<3>{\import{assets/}{latticeB_3.pdf_tex}}% + \only<4>{\import{assets/}{latticeB_4.pdf_tex}}% + \only<5>{\import{assets/}{latticeB_5.pdf_tex}}% + \only<6>{\import{assets/}{latticeB_6.pdf_tex}}% + \only<7>{\import{assets/}{latticeB_7.pdf_tex}}% + \only<8>{\import{assets/}{latticeB_8.pdf_tex}}% + \only<9>{\import{assets/}{latticeB_9.pdf_tex}}% + \only<10>{\import{assets/}{latticeB_10.pdf_tex}}% + \end{figure} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: monotonic-reads consistency} + \textbf{Property:} If node $A$ did an operation $read()$ and received $x$ as a response,\\ + \hspace{2cm} and node $B$ starts an operation $read()$ after $A$ received $x$,\\ + \hspace{2cm} then $B$ will read a value $x' \sqsupseteq x$. + + \vspace{1em} + + \textbf{Algorithm $monotonic\_read()$:} {\small (a.k.a. repair-on-read)} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply with values $x_1, \dots, x_k$ + \item If $x_i \ne x_j$ for some nodes $i$ and $j$,\\ + \hspace{1cm}then call $write(x_1 \sqcup \dots \sqcup x_k)$ and wait for OK from $k' > n/2$ nodes + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + + \vspace{1em} + + This makes reads slower in some cases, and is \textbf{not implemented in Garage}. +\end{frame} + +\begin{frame} + \frametitle{A hard problem: layout changes} + \begin{itemize} + \item We rely on quorums $k > n/2$ within each partition:\\ + $$n=3,~~~~~~~k\ge 2$$ + \item<2-> When rebalancing, the set of nodes responsible for a partition can change:\\ + $$\{n_A, n_B, n_C\} \to \{n_A, n_D, n_E\}$$ + \vspace{.01em} + \item<3-> During the rebalancing, $D$ and $E$ don't yet have the data,\\ + ~~~~~~~~~~~~~~~~~~~and $B$ and $C$ want to get rid of the data to free up space\\ + \vspace{.2em} + $\to$ quorums only within the new set of nodes don't work\\ + $\to$ how to coordinate? \textbf{currently, we don't...} + + \end{itemize} +\end{frame} + +\section{Operating big Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{assets/scr_garage_status.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.85\linewidth]{assets/scr_garage_status_broken.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{assets/garage.drawio.pdf}}% + \only<2>{\includegraphics[width=.6\linewidth]{assets/garage_sync.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/scr_garage_stats.png}} + \only<2>{\includegraphics[width=.6\linewidth]{assets/scr_garage_worker_list.png}} + \only<3>{\includegraphics[width=.6\linewidth]{assets/scr_garage_worker_get.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \item Multi-HDD support not yet released (soon!) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Wait for v0.9 with multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Current deployments: $< 10$ TB, we don't have much experience with more +\end{frame} + +\section{Going further than the S3 API} + +\begin{frame} + \frametitle{Using Garage for everything} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/slideB1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{assets/slideB2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{assets/slideB3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{K2V Design} + \begin{itemize} + \item A new, custom, minimal API\\ + \vspace{.5em} + \begin{itemize} + \item Single-item operations + \item Operations on ranges and batches of items + \item Polling operations to help implement a PubSub pattern + \end{itemize} + \vspace{1em} + \item<2-> Exposes the partitoning mechanism of Garage\\ + K2V = partition key / sort key / value (like Dynamo) + \vspace{1em} + \item<3-> Weakly consistent, CRDT-friendly\\ + $\to$ no support for transactions (not ACID) + \vspace{1em} + \item<4-> Cryptography-friendly: values are binary blobs + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Handling concurrent values} + \textbf{How to handle concurrency?} Example: + \vspace{1em} + \begin{enumerate} + \item Client $A$ reads the initial value of a key, $x_0$ + \vspace{1em} + \item<2-> Client $B$ also reads the initial value $x_0$ of that key + \vspace{1em} + \item<3-> Client $A$ modifies $x_0$, and writes a new value $x_1$ + \vspace{1em} + \item<4-> Client $B$ also modifies $x_0$, and writes a new value $x'_1$,\\ + without having a chance to first read $x_1$\\ + \vspace{1em} + $\to$ what should the final state be? + \end{enumerate} +\end{frame} + +\begin{frame} + \frametitle{Handling concurrent values} + \begin{itemize} + \item If we keep only $x_1$ or $x'_1$, we risk \textbf{loosing application data} + \vspace{1.5em} + \item<2-> Values are opaque binary blobs, \textbf{K2V cannot resolve conflicts} by itself\\ + (e.g. by implementing a CRDT) + \vspace{1.5em} + \item<3-> Solution: \textbf{we keep both!}\\ + $\to$ the value of the key is now $\{x_1, x'_1\}$\\ + $\to$ the client application can decide how to resolve conflicts on the next read + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Keeping track of causality} + How does K2V know that $x_1$ and $x'_1$ are concurrent? + \vspace{1em} + \begin{itemize} + \item $read()$ returns \textbf{a set of values} and an associated \textbf{causality token}\\ + \vspace{1.5em} + \item<2-> When calling $write()$, the client sends \textbf{the causality token from its last read} + \vspace{1.5em} + \item<3-> The causality token represents the set of values \textbf{already seen by the client}\\ + $\to$ those values are the \textbf{causal past} of the write operation\\ + $\to$ K2V can keep concurrent values and overwrite all ones in the causal past + \vspace{1.5em} + \item<4-> Internally, the causality token is \textbf{a vector clock} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Application: an e-mail storage server} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Aerogramme data model} + \begin{center} + \only<1->{\includegraphics[width=.4\linewidth]{assets/aerogramme_datatype.drawio.pdf}}% + \end{center} + \visible<2->{Aerogramme encrypts all stored values for privacy\\ + (Garage server administrators can't read your mail)} +\end{frame} + +\begin{frame} + \frametitle{Different deployment scenarios} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components1.drawio.pdf}}% + \only<2>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components2.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{A new model for building resilient software} + How to build an application using only Garage as a data store: + \vspace{1em} + \begin{enumerate} + \item Design a data model suited to K2V\\ + {\footnotesize (see Cassandra docs on porting SQL data models to Cassandra)} + \vspace{1em} + \begin{itemize} + \item Use CRDTs or other eventually consistent data types (see e.g. Bayou) + \vspace{1em} + \item Store opaque binary blobs to provide End-to-End Encryption\\ + \end{itemize} + \vspace{1em} + \item<2-> Store big blobs (files) using the S3 API + \vspace{1em} + \item<3-> Let Garage manage sharding, replication, failover, etc. + \end{enumerate} +\end{frame} + +\section{Conclusion} + +\begin{frame} + \frametitle{Perspectives} + \begin{itemize} + \item Fix the consistency issue when rebalancing + \vspace{1em} + \item Write about Garage's architecture and properties,\\ + and about our proposed architecture for (E2EE) apps over K2V+S3 + \vspace{1em} + \item Continue developing Garage; finish Aerogramme; build new applications... + \vspace{1em} + \item Anything else? + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{assets/rust_logo.png} + \includegraphics[width=.13\linewidth]{assets/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-01-12-seed/.gitignore b/doc/talks/2024-01-12-seed/.gitignore new file mode 100644 index 00000000..b7e0ced2 --- /dev/null +++ b/doc/talks/2024-01-12-seed/.gitignore @@ -0,0 +1,10 @@ +* + +!*.txt +!*.md +!*.tex + +!talk.pdf +!Makefile +!.gitignore + diff --git a/doc/talks/2024-01-12-seed/Makefile b/doc/talks/2024-01-12-seed/Makefile new file mode 100644 index 00000000..5f0befbe --- /dev/null +++ b/doc/talks/2024-01-12-seed/Makefile @@ -0,0 +1,10 @@ +ASSETS=../assets/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +assets/%.pdf: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +assets/%.pdf_tex: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-01-12-seed/talk.pdf b/doc/talks/2024-01-12-seed/talk.pdf new file mode 100644 index 00000000..85c3bcde Binary files /dev/null and b/doc/talks/2024-01-12-seed/talk.pdf differ diff --git a/doc/talks/2024-01-12-seed/talk.tex b/doc/talks/2024-01-12-seed/talk.tex new file mode 100644 index 00000000..cd6c4681 --- /dev/null +++ b/doc/talks/2024-01-12-seed/talk.tex @@ -0,0 +1,370 @@ +\nonstopmode +\documentclass[aspectratio=169]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\subtitle{a lightweight and robust geo-distributed data storage system} +\author{Alex Auvolat, Deuxfleurs} +\date{SEED webinar, 2024-01-12} + +\begin{document} + +% \begin{frame} +% \centering +% \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} +% \vspace{1em} +% +% {\large\bf Alex Auvolat, Deuxfleurs Association} +% \vspace{1em} +% +% \url{https://garagehq.deuxfleurs.fr/} +% +% %Matrix channel: \texttt{\#garage:deuxfleurs.fr} +% \end{frame} + +\begin{frame} + %\frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + Member of Deuxfleurs, lead developer of Garage + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{.5em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.6\linewidth, valign=t]{../../logo/garage-notext.png} + \end{column} + \begin{column}{.6\textwidth} + \\\textbf{Garage}\\ + A self-hosted alternative to S3 for object storage + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Stable vs Resilient} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Building a "stable" system:} + \vspace{1em} + + Enterprise-grade systems typically employ: + \vspace{1em} + \begin{itemize} + \item RAID + \item Redundant power grid + UPS + \item Redundant Internet connections + \item Low-latency links + \item ... + \end{itemize} + \vspace{1em} + $\to$ costly, only worth at DC scale\\ + $\to$ still risk of DC-level incident... + \end{minipage} + \hfill + \begin{minipage}{7cm} + \textbf{Building a \underline{resilient} system:} + \vspace{1em} + + An alternative, cheaper way: + \vspace{1em} + \begin{itemize} + \item Commodity hardware \\(e.g. old desktop PCs) + \vspace{.5em} + \item Commodity Internet \\(e.g. FTTB, FTTH) and power grid + \vspace{.5em} + \item \textbf{Geographical redundancy} \\(multi-site replication) + \end{itemize} + \vspace{1.5em} + \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Example: our infrastructure at Deuxfleurs} + \only<1>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/atuin.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Object storage: simpler than file systems} + + \begin{minipage}{6cm} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! + \end{minipage} + \hfill + \begin{minipage}{8cm} + \begin{center} + \vspace{2em} + \includegraphics[height=6em]{../2020-12-02_wide-team/img/Amazon-S3.jpg} + \hspace{2em} + \includegraphics[height=5em]{../assets/minio.png} + + \vspace{2em} + \includegraphics[height=6em]{../../logo/garage_hires_crop.png} + \end{center} + \vspace{1em} + \end{minipage} +\end{frame} + + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a key-value store: + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|p{8cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Implementation: consensus vs weak consistency} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Consensus-based systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Leader-based:} a leader is elected to coordinate + all reads and writes + \vspace{1em} + \item Allows for \textbf{sequential reasoning}: + program as if running on a single machine + \vspace{1em} + \item Serializability is one of the \\ + \textbf{strongest consistency guarantees} + \vspace{1em} + \item \textbf{Costly}, the leader is a bottleneck; + leader elections on failure take time + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}{7cm} \visible<2->{ + \textbf{Weakly consistent systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Nodes are equivalent}, any node + can originate a read or write operation + \vspace{1em} + \item \textbf{Operations must be independent}, + conflicts are resolved after the fact + \vspace{1em} + \item Strongest achievable consistency:\\ + \textbf{read-after-write consistency}\\(using quorums) + \vspace{1em} + \item \textbf{Fast}, no single bottleneck;\\ + works transparently with offline nodes + \end{itemize} + } \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Why avoid consensus?} + Consensus can be implemented reasonably well in practice, so why avoid it? + \vspace{2em} + \begin{itemize} + \item \textbf{Software complexity:} RAFT and PAXOS are complex beasts;\\ + harder to prove, harder to reason about + \vspace{1.5em} + \item \textbf{Performance issues:} + \vspace{1em} + \begin{itemize} + \item Taking a decision may take an \textbf{arbitrary number of steps} (in adverse scenarios) + \vspace{1em} + \item The leader is a \textbf{bottleneck} for all requests;\\ + even in leaderless approaches, \textbf{all nodes must process all operations in order} + \vspace{1em} + \item Particularly \textbf{sensitive to higher latency} between nodes + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Objective: the right level of consistency for Garage} + + \underline{Constraints:} slow network (geographical distance), node unavailability/crashes\\ + \underline{Objective:} maximize availability, maintain an \emph{appropriate level of consistency}\\ + \vspace{1em} + \begin{enumerate} + \item<2-> \textbf{Weak consistency for most things}\\ + \vspace{1em} + \underline{Example:} \texttt{PutObject}\\ + \vspace{.5em} + If two clients write the same + object at the same time, one of the two is implicitly overwritten. + No need to coordinate, use a \emph{last-writer-wins register}. + \vspace{1em} + \item<3-> \textbf{Stronger consistency only when necessary}\\ + \vspace{1em} + \underline{Example:} \texttt{CreateBucket}\\ + \vspace{.5em} + A bucket is a reserved name in a shared namespace, + two clients should be prevented from both creating the same bucket + (\emph{mutual exclusion}). + \end{enumerate} +\end{frame} + +\begin{frame} + \frametitle{The possibility of \emph{leaderless consensus}} + Currently, Garage \emph{only has weak consistency}. Is fast, but \texttt{CreateBucket} is broken! + + \visible<2->{ + \vspace{1em} + Leaderless consensus (Antoniadis et al., 2023) alleviates issues with RAFT and PAXOS: + \vspace{1em} + \begin{itemize} + \item \textbf{No leader.} All nodes participate equally at each time step, + and different nodes can be unavailable at different times without issues. + \\ \vspace{.5em} $\to$ better tolerance to the high latency (remove bottleneck issue) + \\ $\to$ tolerates crash transparently + \vspace{1em} + \item \textbf{Simpler formalization.} The algorithm is very simple to express and to analyze in mathematical terms. + \end{itemize} + } + \visible<3->{ + \vspace{1em} + One of the possible subjects for this PhD: + \\$\to$ \emph{integration of leaderless consensus in Garage} + testing + perf eval, etc. + } +\end{frame} + +\begin{frame} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-02-03-fosdem/.gitignore b/doc/talks/2024-02-03-fosdem/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2024-02-03-fosdem/Makefile b/doc/talks/2024-02-03-fosdem/Makefile new file mode 100644 index 00000000..f0aae6a8 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/Makefile @@ -0,0 +1,19 @@ +ASSETS=../assets/lattice/lattice1.pdf_tex \ + ../assets/lattice/lattice2.pdf_tex \ + ../assets/lattice/lattice3.pdf_tex \ + ../assets/lattice/lattice4.pdf_tex \ + ../assets/lattice/lattice5.pdf_tex \ + ../assets/lattice/lattice6.pdf_tex \ + ../assets/lattice/lattice7.pdf_tex \ + ../assets/lattice/lattice8.pdf_tex \ + ../assets/logos/deuxfleurs.pdf \ + ../assets/timeline-22-24.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-02-03-fosdem/talk.pdf b/doc/talks/2024-02-03-fosdem/talk.pdf new file mode 100644 index 00000000..ef3f5c81 Binary files /dev/null and b/doc/talks/2024-02-03-fosdem/talk.pdf differ diff --git a/doc/talks/2024-02-03-fosdem/talk.tex b/doc/talks/2024-02-03-fosdem/talk.tex new file mode 100644 index 00000000..b36bf893 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/talk.tex @@ -0,0 +1,764 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage, the low-tech storage platform for geo-distributed clusters} +\author{Alex Auvolat, Deuxfleurs} +\date{FOSDEM'24, 2024-02-03} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Building a resilient system with cheap stuff} + + \only<1,4-7>{ + \begin{itemize} + \item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<4->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<6->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<7-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/atuin.jpg} + \end{center} + } + \only<8>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Object storage: a crucial component} + \begin{center} + \includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{1em} + \begin{itemize} + \item<2> Maps well to CRDT data types + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + + +% ======================================== TIMELINE +% ======================================== TIMELINE +% ======================================== TIMELINE + +\section{Recent developments} + +% ====================== v0.7.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{April 2022 - Garage v0.7.0} + Focus on \underline{observability and ecosystem integration} + \vspace{2em} + \begin{itemize} + \item \textbf{Monitoring:} metrics and traces, using OpenTelemetry + \vspace{1em} + \item Replication modes with 1 or 2 copies / weaker consistency + \vspace{1em} + \item Kubernetes integration for node discovery + \vspace{1em} + \item Admin API (v0.7.2) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Metrics (Prometheus + Grafana)} + \begin{center} + \includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Traces (Jaeger)} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ====================== v0.8.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{November 2022 - Garage v0.8.0} + Focus on \underline{performance} + \vspace{2em} + \begin{itemize} + \item \textbf{Alternative metadata DB engines} (LMDB, Sqlite) + \vspace{1em} + \item \textbf{Performance improvements:} block streaming, various optimizations... + \vspace{1em} + \item Bucket quotas (max size, max \#objects) + \vspace{1em} + \item Quality of life improvements, observability, etc. + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{About metadata DB engines} + \textbf{Issues with Sled:} + \vspace{1em} + \begin{itemize} + \item Huge files on disk + \vspace{.5em} + \item Unpredictable performance, especially on HDD + \vspace{.5em} + \item API limitations + \vspace{.5em} + \item Not actively maintained + \end{itemize} + + \vspace{2em} + \textbf{LMDB:} very stable, good performance, file size is reasonable\\ + \textbf{Sqlite} also available as a second choice + + \vspace{1em} + Sled will be removed in Garage v1.0 +\end{frame} + +\begin{frame} + \frametitle{DB engine performance comparison} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/perf/db_engine.png} + \end{center} + NB: Sqlite was slow due to synchronous mode, now configurable +\end{frame} + +\begin{frame} + \frametitle{Block streaming} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-1.png}} + \only<2>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-2.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{TTFB benchmark} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/ttfb.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Throughput benchmark} + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/perf/io-0.7-0.8-minio.png} + \end{center} +\end{frame} + +% ====================== v0.9.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.9.0} + Focus on \underline{streamlining \& usability} + \vspace{2em} + \begin{itemize} + \item Support multiple HDDs per node + \vspace{1em} + \item S3 compatibility: + \vspace{1em} + \begin{itemize} + \item support basic lifecycle configurations + \vspace{.5em} + \item allow for multipart upload part retries + \end{itemize} + \vspace{1em} + \item LMDB by default, deprecation of Sled + \vspace{1em} + \item New layout computation algorithm + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.10.0 beta} + Focus on \underline{consistency} + \vspace{2em} + \begin{itemize} + \item Fix consistency issues when reshuffling data + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Working with weak consistency} + Not using consensus limits us to the following: + \vspace{2em} + \begin{itemize} + \item<2-> \textbf{Conflict-free replicated data types} (CRDT)\\ + \vspace{1em} + {\footnotesize Non-transactional key-value stores such as S3 are equivalent to a simple CRDT:\\ + a map of \textbf{last-writer-wins registers} (each key is its own CRDT)} + \vspace{1.5em} + \item<3-> \textbf{Read-after-write consistency}\\ + \vspace{1em} + {\footnotesize Can be implemented using quorums on read and write operations} + \end{itemize} +\end{frame} + +\begin{frame}[t] + \frametitle{CRDT read-after-write consistency using quorums} + + \vspace{1em} + {\small + \textbf{Property:} If client 1 did an operation $write(x)$ and received an OK response,\\ + \hspace{2cm} and client 2 starts an operation $read()$ after client 1 received OK,\\ + \hspace{2cm} then client 2 will read a value $x' \sqsupseteq x$. + } + + \vspace{1.5em} + \begin{overprint} + \onslide<2-9> + \begin{figure} + \centering + \footnotesize + \def\svgwidth{.7\textwidth} + \only<2>{\import{../assets/lattice/}{lattice1.pdf_tex}}% + \only<3>{\import{../assets/lattice/}{lattice2.pdf_tex}}% + \only<4>{\import{../assets/lattice/}{lattice3.pdf_tex}}% + \only<5>{\import{../assets/lattice/}{lattice4.pdf_tex}}% + \only<6>{\import{../assets/lattice/}{lattice5.pdf_tex}}% + \only<7>{\import{../assets/lattice/}{lattice6.pdf_tex}}% + \only<8>{\import{../assets/lattice/}{lattice7.pdf_tex}}% + \only<9>{\import{../assets/lattice/}{lattice8.pdf_tex}}% + \end{figure} + + \onslide<10> + \begin{minipage}{.10\textwidth} + ~ + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \textbf{Algorithm $write(x)$:} + \begin{enumerate} + \item Broadcast $write(x)$ to all nodes + \item Wait for $k > n/2$ nodes to reply OK + \item Return OK + \end{enumerate} + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \vspace{1em} + \textbf{Algorithm $read()$:} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply\\ + with values $x_1, \dots, x_k$ + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + \end{minipage} + \end{overprint} +\end{frame} + +\begin{frame} + \frametitle{A hard problem: layout changes} + \begin{itemize} + \item We rely on quorums $k > n/2$ within each partition:\\ + $$n=3,~~~~~~~k\ge 2$$ + \item<2-> When rebalancing, the set of nodes responsible for a partition can change:\\ + + \vspace{1em} + \begin{minipage}{.04\linewidth}~ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{Crimson}{df-ymk} & Abricot & \textcolor{Crimson}{Courgette} \\ + \hline + Partition 1 & Ananas & \textcolor{Crimson}{Courgette} & \textcolor{Crimson}{df-ykl} \\ + \hline + Partition 2 & \textcolor{Crimson}{df-ymf} & \textcolor{Crimson}{Celeri} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + \begin{minipage}{.04\linewidth} + $\to$ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{ForestGreen}{Dahlia} & Abricot & \textcolor{ForestGreen}{Eucalyptus} \\ + \hline + Partition 1 & Ananas & \textcolor{ForestGreen}{Euphorbe} & \textcolor{ForestGreen}{Doradille} \\ + \hline + Partition 2 & \textcolor{ForestGreen}{Dahlia} & \textcolor{ForestGreen}{Echinops} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + + \vspace{2em} + \item<3-> During the rebalancing, new nodes don't yet have the data,\\ + ~~~~~~~~~~~~~~~~~~~and old nodes want to get rid of the data to free up space\\ + \vspace{1.2em} + $\to$ risk of inconsistency, \textbf{how to coordinate?} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Handling layout changes without losing consistency} + \begin{minipage}{.55\textwidth} + \begin{itemize} + \item \textbf{Solution:}\\ + \vspace{.5em} + \begin{itemize} + \item keep track of data transfer to new nodes + \vspace{.5em} + \item use multiple write quorums\\ + (new nodes + old nodes\\ + while data transfer is in progress) + \vspace{.5em} + \item switching reads to new nodes\\ + only once copy is finished + \end{itemize} + \vspace{1em} + \item \textbf{Implemented} in v0.10 + \vspace{1em} + \item \textbf{Validated} with Jepsen testing + \end{itemize} + \end{minipage} + \begin{minipage}{.23\textwidth} + \includegraphics[width=3cm]{../assets/jepsen-0.9.png}\\ + {\footnotesize Garage v0.9.0} + \end{minipage} + \begin{minipage}{.2\textwidth} + \includegraphics[width=3cm]{../assets/jepsen-0.10.png}\\ + {\footnotesize Garage v0.10 beta} + \end{minipage} +\end{frame} + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Towards v1.0...} + Focus on \underline{security \& stability} + \vspace{2em} + \begin{itemize} + \item \textbf{Security audit} in progress by Radically Open Security + \vspace{1em} + \item Misc. S3 features (SSE-C, ...) and compatibility fixes + \vspace{1em} + \item Improve UX + \vspace{1em} + \item Fix bugs + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{...and beyond!} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/survey_requested_features.png} + \end{center} +\end{frame} + +% ======================================== OPERATING +% ======================================== OPERATING +% ======================================== OPERATING + + +\section{Operating big Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{../assets/garage.drawio.pdf}}% + \only<2>{\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-02-29-capitoul/.gitignore b/doc/talks/2024-02-29-capitoul/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2024-02-29-capitoul/Makefile b/doc/talks/2024-02-29-capitoul/Makefile new file mode 100644 index 00000000..aa9c35af --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/Makefile @@ -0,0 +1,10 @@ +ASSETS=../assets/logos/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-02-29-capitoul/talk.pdf b/doc/talks/2024-02-29-capitoul/talk.pdf new file mode 100644 index 00000000..33a63e23 Binary files /dev/null and b/doc/talks/2024-02-29-capitoul/talk.pdf differ diff --git a/doc/talks/2024-02-29-capitoul/talk.tex b/doc/talks/2024-02-29-capitoul/talk.tex new file mode 100644 index 00000000..95122b24 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/talk.tex @@ -0,0 +1,543 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\author{Alex Auvolat, Deuxfleurs} +\date{Capitoul, 2024-02-29} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Our very low-tech infrastructure} + + \only<1,3-6>{ + \begin{itemize} + \item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<3->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<5->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<6-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<7>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec (e.g.~locks) + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +% --------- BASED ON CRDTS ---------- + +\section{Principle 1: based on CRDTs} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{.5em} + \begin{itemize} + \item<2-> Maps well to CRDT data types + \item<3> Read-after-write consistency with quorums + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + +% --------- GEO-DISTRIBUTED MODEL ---------- + +\section{Principle 2: geo-distributed data model} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \textbf{Two keys:} + \begin{itemize} + \item Partition key: used to divide data into partitions {\small (a.k.a.~shards)} + \item Sort key: used to identify items inside a partition + \end{itemize} + + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|l|p{3cm}|} + \hline + \textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & \texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + +\begin{frame} + \frametitle{The relationship between \emph{partition} and \emph{partition key}} + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} + \vspace{1em} + \textbf{To read or write an item:} hash partition key + \\ \hspace{5cm} $\to$ determine partition number (first 8 bits) + \\ \hspace{5cm} $\to$ find associated nodes +\end{frame} + +\begin{frame} + \frametitle{Garage's internal data structures} + \centering + \includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf} +\end{frame} + +% ---------- OPERATING GARAGE --------- + +\section{Operating Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Background synchronization} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Monitoring with Prometheus + Grafana} + \begin{center} + \includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Debugging with traces} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ---------- SCALING GARAGE --------- + +\section{Scaling Garage clusters} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/assets/.gitignore b/doc/talks/assets/.gitignore new file mode 100644 index 00000000..9ce0997d --- /dev/null +++ b/doc/talks/assets/.gitignore @@ -0,0 +1,8 @@ +# Files that are auto-generated when building pdfs +deuxfleurs.pdf +timeline-22-24.pdf +lattice*.pdf_tex +lattice*.pdf + +# tmp files generated by krita +*~ diff --git a/doc/talks/assets/alex.jpg b/doc/talks/assets/alex.jpg new file mode 100644 index 00000000..eac0f0a9 Binary files /dev/null and b/doc/talks/assets/alex.jpg differ diff --git a/doc/talks/assets/atuin.jpg b/doc/talks/assets/atuin.jpg new file mode 100644 index 00000000..f2fbd61d Binary files /dev/null and b/doc/talks/assets/atuin.jpg differ diff --git a/doc/talks/assets/garage.drawio.pdf b/doc/talks/assets/garage.drawio.pdf new file mode 100644 index 00000000..a54a163c Binary files /dev/null and b/doc/talks/assets/garage.drawio.pdf differ diff --git a/doc/talks/assets/garage.drawio.png b/doc/talks/assets/garage.drawio.png new file mode 100644 index 00000000..386dd862 Binary files /dev/null and b/doc/talks/assets/garage.drawio.png differ diff --git a/doc/talks/assets/garage2.drawio.png b/doc/talks/assets/garage2.drawio.png new file mode 100644 index 00000000..8562fbcf Binary files /dev/null and b/doc/talks/assets/garage2.drawio.png differ diff --git a/doc/talks/assets/garage2a.drawio.pdf b/doc/talks/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..422c9343 Binary files /dev/null and b/doc/talks/assets/garage2a.drawio.pdf differ diff --git a/doc/talks/assets/garage2b.drawio.pdf b/doc/talks/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..05a9710e Binary files /dev/null and b/doc/talks/assets/garage2b.drawio.pdf differ diff --git a/doc/talks/assets/garage_sync.drawio.pdf b/doc/talks/assets/garage_sync.drawio.pdf new file mode 100644 index 00000000..a94b3572 Binary files /dev/null and b/doc/talks/assets/garage_sync.drawio.pdf differ diff --git a/doc/talks/assets/garage_sync.drawio.png b/doc/talks/assets/garage_sync.drawio.png new file mode 100644 index 00000000..2e7b5af0 Binary files /dev/null and b/doc/talks/assets/garage_sync.drawio.png differ diff --git a/doc/talks/assets/garage_tables.pdf b/doc/talks/assets/garage_tables.pdf new file mode 100644 index 00000000..3c54cdce Binary files /dev/null and b/doc/talks/assets/garage_tables.pdf differ diff --git a/doc/talks/assets/geodistrib_paper.png b/doc/talks/assets/geodistrib_paper.png new file mode 100644 index 00000000..318d8e55 Binary files /dev/null and b/doc/talks/assets/geodistrib_paper.png differ diff --git a/doc/talks/assets/inframap_jdll2023.pdf b/doc/talks/assets/inframap_jdll2023.pdf new file mode 100644 index 00000000..77f97baa Binary files /dev/null and b/doc/talks/assets/inframap_jdll2023.pdf differ diff --git a/doc/talks/assets/intro/slide1.png b/doc/talks/assets/intro/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/assets/intro/slide1.png differ diff --git a/doc/talks/assets/intro/slide2.png b/doc/talks/assets/intro/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/assets/intro/slide2.png differ diff --git a/doc/talks/assets/intro/slide3.png b/doc/talks/assets/intro/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/assets/intro/slide3.png differ diff --git a/doc/talks/assets/intro/slideB1.png b/doc/talks/assets/intro/slideB1.png new file mode 100644 index 00000000..b14b6070 Binary files /dev/null and b/doc/talks/assets/intro/slideB1.png differ diff --git a/doc/talks/assets/intro/slideB2.png b/doc/talks/assets/intro/slideB2.png new file mode 100644 index 00000000..a881a796 Binary files /dev/null and b/doc/talks/assets/intro/slideB2.png differ diff --git a/doc/talks/assets/intro/slideB3.png b/doc/talks/assets/intro/slideB3.png new file mode 100644 index 00000000..830709d2 Binary files /dev/null and b/doc/talks/assets/intro/slideB3.png differ diff --git a/doc/talks/assets/intro/slides.svg b/doc/talks/assets/intro/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/assets/intro/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/assets/intro/slidesB.svg b/doc/talks/assets/intro/slidesB.svg new file mode 100644 index 00000000..c0a6e97c --- /dev/null +++ b/doc/talks/assets/intro/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database diff --git a/doc/talks/assets/jepsen-0.10.png b/doc/talks/assets/jepsen-0.10.png new file mode 100644 index 00000000..d7895e31 Binary files /dev/null and b/doc/talks/assets/jepsen-0.10.png differ diff --git a/doc/talks/assets/jepsen-0.9.png b/doc/talks/assets/jepsen-0.9.png new file mode 100644 index 00000000..3f654038 Binary files /dev/null and b/doc/talks/assets/jepsen-0.9.png differ diff --git a/doc/talks/assets/lattice/lattice1.svg b/doc/talks/assets/lattice/lattice1.svg new file mode 100644 index 00000000..8bfa5aa7 --- /dev/null +++ b/doc/talks/assets/lattice/lattice1.svg @@ -0,0 +1,433 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + diff --git a/doc/talks/assets/lattice/lattice2.svg b/doc/talks/assets/lattice/lattice2.svg new file mode 100644 index 00000000..adcd92cb --- /dev/null +++ b/doc/talks/assets/lattice/lattice2.svg @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice3.svg b/doc/talks/assets/lattice/lattice3.svg new file mode 100644 index 00000000..640dc468 --- /dev/null +++ b/doc/talks/assets/lattice/lattice3.svg @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice4.svg b/doc/talks/assets/lattice/lattice4.svg new file mode 100644 index 00000000..b2a99e28 --- /dev/null +++ b/doc/talks/assets/lattice/lattice4.svg @@ -0,0 +1,525 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice5.svg b/doc/talks/assets/lattice/lattice5.svg new file mode 100644 index 00000000..bc6b7195 --- /dev/null +++ b/doc/talks/assets/lattice/lattice5.svg @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice6.svg b/doc/talks/assets/lattice/lattice6.svg new file mode 100644 index 00000000..176b1715 --- /dev/null +++ b/doc/talks/assets/lattice/lattice6.svg @@ -0,0 +1,553 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice7.svg b/doc/talks/assets/lattice/lattice7.svg new file mode 100644 index 00000000..7ce8bda8 --- /dev/null +++ b/doc/talks/assets/lattice/lattice7.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice8.svg b/doc/talks/assets/lattice/lattice8.svg new file mode 100644 index 00000000..c94a69b2 --- /dev/null +++ b/doc/talks/assets/lattice/lattice8.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/assets/logos/AGPLv3_Logo.png b/doc/talks/assets/logos/AGPLv3_Logo.png new file mode 100644 index 00000000..445284a3 Binary files /dev/null and b/doc/talks/assets/logos/AGPLv3_Logo.png differ diff --git a/doc/talks/assets/logos/Amazon-S3.jpg b/doc/talks/assets/logos/Amazon-S3.jpg new file mode 100644 index 00000000..a9501973 Binary files /dev/null and b/doc/talks/assets/logos/Amazon-S3.jpg differ diff --git a/doc/talks/assets/logos/NGI0Entrust_tag.png b/doc/talks/assets/logos/NGI0Entrust_tag.png new file mode 100644 index 00000000..064c4aee Binary files /dev/null and b/doc/talks/assets/logos/NGI0Entrust_tag.png differ diff --git a/doc/talks/assets/logos/NGI0Entrust_tag.svg b/doc/talks/assets/logos/NGI0Entrust_tag.svg new file mode 100644 index 00000000..ca0d2814 --- /dev/null +++ b/doc/talks/assets/logos/NGI0Entrust_tag.svg @@ -0,0 +1,149 @@ + + + +image/svg+xml + + + + + + +NGI Zero Entrust + + + + diff --git a/doc/talks/assets/logos/capitole_du_libre.png b/doc/talks/assets/logos/capitole_du_libre.png new file mode 100644 index 00000000..eaa521af Binary files /dev/null and b/doc/talks/assets/logos/capitole_du_libre.png differ diff --git a/doc/talks/assets/logos/deuxfleurs.svg b/doc/talks/assets/logos/deuxfleurs.svg new file mode 100644 index 00000000..c298c22b --- /dev/null +++ b/doc/talks/assets/logos/deuxfleurs.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + D + F + diff --git a/doc/talks/assets/logos/fosdem22.png b/doc/talks/assets/logos/fosdem22.png new file mode 100644 index 00000000..17332e4b Binary files /dev/null and b/doc/talks/assets/logos/fosdem22.png differ diff --git a/doc/talks/assets/logos/fosdem24.png b/doc/talks/assets/logos/fosdem24.png new file mode 100644 index 00000000..11631695 Binary files /dev/null and b/doc/talks/assets/logos/fosdem24.png differ diff --git a/doc/talks/assets/logos/jdll.png b/doc/talks/assets/logos/jdll.png new file mode 100644 index 00000000..a7cbe091 Binary files /dev/null and b/doc/talks/assets/logos/jdll.png differ diff --git a/doc/talks/assets/logos/logo_chatons.png b/doc/talks/assets/logos/logo_chatons.png new file mode 100644 index 00000000..890cf17e Binary files /dev/null and b/doc/talks/assets/logos/logo_chatons.png differ diff --git a/doc/talks/assets/logos/minio.png b/doc/talks/assets/logos/minio.png new file mode 100644 index 00000000..a71e9ccc Binary files /dev/null and b/doc/talks/assets/logos/minio.png differ diff --git a/doc/talks/assets/logos/ngi-pointer.png b/doc/talks/assets/logos/ngi-pointer.png new file mode 100644 index 00000000..730ececa Binary files /dev/null and b/doc/talks/assets/logos/ngi-pointer.png differ diff --git a/doc/talks/assets/logos/nlnet.svg b/doc/talks/assets/logos/nlnet.svg new file mode 100644 index 00000000..373c8d8f --- /dev/null +++ b/doc/talks/assets/logos/nlnet.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/talks/assets/logos/pses.png b/doc/talks/assets/logos/pses.png new file mode 100644 index 00000000..da871f83 Binary files /dev/null and b/doc/talks/assets/logos/pses.png differ diff --git a/doc/talks/assets/logos/rust_logo.png b/doc/talks/assets/logos/rust_logo.png new file mode 100644 index 00000000..0e4809ec Binary files /dev/null and b/doc/talks/assets/logos/rust_logo.png differ diff --git a/doc/talks/assets/map.png b/doc/talks/assets/map.png new file mode 100644 index 00000000..1dff3ab6 Binary files /dev/null and b/doc/talks/assets/map.png differ diff --git a/doc/talks/assets/neptune.jpg b/doc/talks/assets/neptune.jpg new file mode 100644 index 00000000..61fcbff6 Binary files /dev/null and b/doc/talks/assets/neptune.jpg differ diff --git a/doc/talks/assets/perf/db_engine.png b/doc/talks/assets/perf/db_engine.png new file mode 100644 index 00000000..b1124b04 Binary files /dev/null and b/doc/talks/assets/perf/db_engine.png differ diff --git a/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png b/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png new file mode 100644 index 00000000..92eac3fb Binary files /dev/null and b/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png differ diff --git a/doc/talks/assets/perf/io-0.7-0.8-minio.png b/doc/talks/assets/perf/io-0.7-0.8-minio.png new file mode 100644 index 00000000..f581a22e Binary files /dev/null and b/doc/talks/assets/perf/io-0.7-0.8-minio.png differ diff --git a/doc/talks/assets/perf/ttfb.png b/doc/talks/assets/perf/ttfb.png new file mode 100644 index 00000000..c0335bd9 Binary files /dev/null and b/doc/talks/assets/perf/ttfb.png differ diff --git a/doc/talks/assets/schema-streaming-1.png b/doc/talks/assets/schema-streaming-1.png new file mode 100644 index 00000000..4113f030 Binary files /dev/null and b/doc/talks/assets/schema-streaming-1.png differ diff --git a/doc/talks/assets/schema-streaming-2.png b/doc/talks/assets/schema-streaming-2.png new file mode 100644 index 00000000..2f97bb33 Binary files /dev/null and b/doc/talks/assets/schema-streaming-2.png differ diff --git a/doc/talks/assets/schema-streaming.png b/doc/talks/assets/schema-streaming.png new file mode 100644 index 00000000..3b24d910 Binary files /dev/null and b/doc/talks/assets/schema-streaming.png differ diff --git a/doc/talks/assets/screenshots/garage_stats_0.10.png b/doc/talks/assets/screenshots/garage_stats_0.10.png new file mode 100644 index 00000000..78a36ead Binary files /dev/null and b/doc/talks/assets/screenshots/garage_stats_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.10.png b/doc/talks/assets/screenshots/garage_status_0.10.png new file mode 100644 index 00000000..510717ca Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.9_prod.png b/doc/talks/assets/screenshots/garage_status_0.9_prod.png new file mode 100644 index 00000000..e1a05899 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.9_prod.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png b/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png new file mode 100644 index 00000000..99d46f98 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png differ diff --git a/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png b/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png new file mode 100644 index 00000000..b167687d Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_worker_list_0.10.png b/doc/talks/assets/screenshots/garage_worker_list_0.10.png new file mode 100644 index 00000000..078f0607 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_worker_list_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_worker_param_0.10.png b/doc/talks/assets/screenshots/garage_worker_param_0.10.png new file mode 100644 index 00000000..39ee11aa Binary files /dev/null and b/doc/talks/assets/screenshots/garage_worker_param_0.10.png differ diff --git a/doc/talks/assets/screenshots/grafana_dashboard.png b/doc/talks/assets/screenshots/grafana_dashboard.png new file mode 100644 index 00000000..bda2d732 Binary files /dev/null and b/doc/talks/assets/screenshots/grafana_dashboard.png differ diff --git a/doc/talks/assets/screenshots/jaeger_listobjects.png b/doc/talks/assets/screenshots/jaeger_listobjects.png new file mode 100644 index 00000000..ab7d68f4 Binary files /dev/null and b/doc/talks/assets/screenshots/jaeger_listobjects.png differ diff --git a/doc/talks/assets/screenshots/jaeger_putobject.png b/doc/talks/assets/screenshots/jaeger_putobject.png new file mode 100644 index 00000000..7966275e Binary files /dev/null and b/doc/talks/assets/screenshots/jaeger_putobject.png differ diff --git a/doc/talks/assets/survey_requested_features.png b/doc/talks/assets/survey_requested_features.png new file mode 100644 index 00000000..07247c46 Binary files /dev/null and b/doc/talks/assets/survey_requested_features.png differ diff --git a/doc/talks/assets/timeline-22-24.svg b/doc/talks/assets/timeline-22-24.svg new file mode 100644 index 00000000..90fe5de4 --- /dev/null +++ b/doc/talks/assets/timeline-22-24.svg @@ -0,0 +1,327 @@ + + + +Feb 6, 2022Feb 3, 2024v0.6.0Feb 2, 2022v0.7.0Apr 8, 2022v0.8.0Nov 21, 2022v0.9.0Oct 10, 2023v0.10.0 betaTowards v1.0 ?Apr/May, 2024Nov 19, 2022Apr 1, 2023PSESJun 6, 2023 diff --git a/doc/talks/shell.nix b/doc/talks/shell.nix index 161a61e1..bbee0714 100644 --- a/doc/talks/shell.nix +++ b/doc/talks/shell.nix @@ -5,7 +5,7 @@ let scheme-basic beamer amsmath mathtools breqn environ - multirow graphics import adjustbox tabu vwcol stmaryrd ulem ragged2e + multirow graphics import adjustbox tabu vwcol stmaryrd ulem ragged2e textpos dvisvgm dvipng wrapfig hyperref capt-of; }); in pkgs.mkShell { nativeBuildInputs = [ pkgs.gnumake latex ]; } diff --git a/flake.lock b/flake.lock index 8de99c3e..211b70e0 100644 --- a/flake.lock +++ b/flake.lock @@ -1,38 +1,27 @@ { "nodes": { - "cargo2nix": { - "inputs": { - "flake-compat": [ - "flake-compat" - ], - "flake-utils": "flake-utils", - "nixpkgs": [ - "nixpkgs" - ], - "rust-overlay": "rust-overlay" - }, + "crane": { "locked": { - "lastModified": 1666087781, - "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=", - "owner": "Alexis211", - "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "lastModified": 1737689766, + "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", + "owner": "ipetkov", + "repo": "crane", + "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", "type": "github" }, "original": { - "owner": "Alexis211", - "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "owner": "ipetkov", + "repo": "crane", "type": "github" } }, "flake-compat": { "locked": { - "lastModified": 1688025799, - "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=", + "lastModified": 1717312683, + "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=", "owner": "nix-community", "repo": "flake-compat", - "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c", + "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea", "type": "github" }, "original": { @@ -46,29 +35,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -79,63 +50,47 @@ }, "nixpkgs": { "locked": { - "lastModified": 1682109806, - "narHash": "sha256-d9g7RKNShMLboTWwukM+RObDWWpHKaqTYXB48clBWXI=", + "lastModified": 1763977559, + "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2362848adf8def2866fabbffc50462e929d7fffb", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1682423271, - "narHash": "sha256-WHhl1GiOij1ob4cTLL+yhqr+vFOUH8E5wAX8Ir8fvjE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "94517a501434a627c5d9e72ac6e7f26174b978d3", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "94517a501434a627c5d9e72ac6e7f26174b978d3", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" } }, "root": { "inputs": { - "cargo2nix": "cargo2nix", + "crane": "crane", "flake-compat": "flake-compat", - "flake-utils": [ - "cargo2nix", - "flake-utils" - ], - "nixpkgs": "nixpkgs_2" + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" } }, "rust-overlay": { "inputs": { - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": [ + "nixpkgs" + ] }, "locked": { - "lastModified": 1682389182, - "narHash": "sha256-8t2nmFnH+8V48+IJsf8AK51ebXNlVbOSVYOpiqJKvJE=", + "lastModified": 1763952169, + "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "74f1a64dd28faeeb85ef081f32cad2989850322c", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" }, "original": { "owner": "oxalica", "repo": "rust-overlay", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" } }, @@ -153,21 +108,6 @@ "repo": "default", "type": "github" } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 32c01616..01a077c4 100644 --- a/flake.nix +++ b/flake.nix @@ -2,52 +2,95 @@ description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; - # Nixpkgs unstable as of 2023-04-25, has rustc v1.68 + # Nixpkgs 25.05 as of 2025-11-24 inputs.nixpkgs.url = - "github:NixOS/nixpkgs/94517a501434a627c5d9e72ac6e7f26174b978d3"; + "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632"; + + # Rust overlay as of 2025-11-24 + inputs.rust-overlay.url = + "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b"; + inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; + + inputs.crane.url = "github:ipetkov/crane"; inputs.flake-compat.url = "github:nix-community/flake-compat"; + inputs.flake-utils.url = "github:numtide/flake-utils"; - inputs.cargo2nix = { - # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection - url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; - - # As of 2023-04-25: - # - my two patches were merged into unstable (one for clippy and one to "fix" feature detection) - # - rustc v1.66 - # url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2"; - - # Rust overlay as of 2023-04-25 - inputs.rust-overlay.url = - "github:oxalica/rust-overlay/74f1a64dd28faeeb85ef081f32cad2989850322c"; - - inputs.nixpkgs.follows = "nixpkgs"; - inputs.flake-compat.follows = "flake-compat"; - }; - - inputs.flake-utils.follows = "cargo2nix/flake-utils"; - - outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }: + outputs = { self, nixpkgs, flake-utils, crane, rust-overlay, ... }: let - git_version = self.lastModifiedDate; compile = import ./nix/compile.nix; in flake-utils.lib.eachDefaultSystem (system: - let pkgs = nixpkgs.legacyPackages.${system}; - in { - packages = { - default = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; - release = true; - }).workspace.garage { compileMode = "build"; }; - }; - devShell = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; + let + pkgs = nixpkgs.legacyPackages.${system}; + packageFor = target: release: (compile { + inherit system target nixpkgs crane rust-overlay release; + }).garage; + testWith = extraTestEnv: (compile { + inherit system nixpkgs crane rust-overlay extraTestEnv; release = false; - }).workspaceShell { packages = [ pkgs.rustfmt ]; }; + }).garage-test; + lints = (compile { + inherit system nixpkgs crane rust-overlay; + release = false; + }); + in + { + packages = { + # default = native release build + default = packageFor null true; + + # = cross-compiled, statically-linked release builds + amd64 = packageFor "x86_64-unknown-linux-musl" true; + i386 = packageFor "i686-unknown-linux-musl" true; + arm64 = packageFor "aarch64-unknown-linux-musl" true; + arm = packageFor "armv6l-unknown-linux-musl" true; + + # dev = native dev build + dev = packageFor null false; + + # test = cargo test + tests = testWith {}; + tests-lmdb = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "lmdb"; + }; + tests-sqlite = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite"; + }; + tests-fjall = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall"; + }; + + # lints (fmt, clippy) + fmt = lints.garage-cargo-fmt; + clippy = lints.garage-cargo-clippy; + }; + + # ---- developpment shell, for making native builds only ---- + devShells = + let + targets = compile { + inherit system nixpkgs crane rust-overlay; + }; + in + { + default = targets.devShell; + + # import the full shell using `nix develop .#full` + full = pkgs.mkShell { + buildInputs = with pkgs; [ + targets.toolchain + protobuf + clang + mold + # ---- extra packages for dev tasks ---- + rust-analyzer + cargo-audit + cargo-outdated + cargo-machete + nixpkgs-fmt + ]; + }; + }; }); } diff --git a/k2v_test.py b/k2v_test.py deleted file mode 100755 index 3219056e..00000000 --- a/k2v_test.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -import os -import requests -from datetime import datetime - -# let's talk to our AWS Elasticsearch cluster -#from requests_aws4auth import AWS4Auth -#auth = AWS4Auth('GK31c2f218a2e44f485b94239e', -# 'b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', -# 'us-east-1', -# 's3') - -from aws_requests_auth.aws_auth import AWSRequestsAuth -auth = AWSRequestsAuth(aws_access_key='GK31c2f218a2e44f485b94239e', - aws_secret_access_key='b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', - aws_host='localhost:3812', - aws_region='us-east-1', - aws_service='k2v') - - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - - -sort_keys = ["a", "b", "c", "d"] - -for sk in sort_keys: - print("-- (%s) Put initial (no CT)"%sk) - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - data='{}: Hello, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- ReadIndex") - response = requests.get('http://localhost:3812/alex', - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put with CT") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Good bye, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put again with same CT (concurrent)") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Concurrent value, oops'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- Delete") - response = requests.delete('http://localhost:3812/alex/root?sort_key=%s'%sk, - headers={'x-garage-causality-token': ct}, - auth=auth) - print(response.headers) - print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -print("-- InsertBatch") -response = requests.post('http://localhost:3812/alex', - auth=auth, - data=''' -[ - {"pk": "root", "sk": "a", "ct": null, "v": "aW5pdGlhbCB0ZXN0Cg=="}, - {"pk": "root", "sk": "b", "ct": null, "v": "aW5pdGlhbCB0ZXN1Cg=="}, - {"pk": "root", "sk": "c", "ct": null, "v": "aW5pdGlhbCB0ZXN2Cg=="} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"}, - {"partitionKey": "root", "tombstones": true}, - {"partitionKey": "root", "tombstones": true, "limit": 2}, - {"partitionKey": "root", "start": "c", "singleItem": true}, - {"partitionKey": "root", "start": "b", "end": "d", "tombstones": true} -] -''') -print(response.headers) -print(response.text) - - -print("-- DeleteBatch") -response = requests.post('http://localhost:3812/alex?delete', - auth=auth, - data=''' -[ - {"partitionKey": "root", "start": "b", "end": "c"} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"} -] -''') -print(response.headers) -print(response.text) diff --git a/nix/build_index.nix b/nix/build_index.nix index 7cc4f62c..7869566f 100644 --- a/nix/build_index.nix +++ b/nix/build_index.nix @@ -2,7 +2,7 @@ with import ./common.nix; let - pkgs = import pkgsSrc { }; + pkgs = import nixpkgs { }; lib = pkgs.lib; /* Converts a key list and a value list to a set diff --git a/nix/common.nix b/nix/common.nix index 323c3882..2c03f04e 100644 --- a/nix/common.nix +++ b/nix/common.nix @@ -10,8 +10,9 @@ let flake = (import flake-compat { system = builtins.currentSystem; src = ../.; }); in -rec { - pkgsSrc = flake.defaultNix.inputs.nixpkgs; - cargo2nix = flake.defaultNix.inputs.cargo2nix; - cargo2nixOverlay = cargo2nix.overlays.default; + +{ + flake = flake.defaultNix; + nixpkgs = flake.defaultNix.inputs.nixpkgs; + devShells = flake.defaultNix.devShells.${builtins.currentSystem}; } diff --git a/nix/compile.nix b/nix/compile.nix index efd88bbf..c6df9dbd 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -1,185 +1,64 @@ -{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc" -, release ? false, git_version ? null, features ? null, }: +{ + /* build inputs */ + nixpkgs, + crane, + rust-overlay, + + /* parameters */ + system, + git_version ? null, + target ? null, + release ? false, + features ? null, + extraTestEnv ? {} +}: let log = v: builtins.trace v v; + # NixOS and Rust/Cargo triples do not match for ARM, fix it here. + rustTarget = if target == "armv6l-unknown-linux-musleabihf" then + "arm-unknown-linux-musleabihf" + else + target; + + rustTargetEnvMap = { + "x86_64-unknown-linux-musl" = "X86_64_UNKNOWN_LINUX_MUSL"; + "aarch64-unknown-linux-musl" = "AARCH64_UNKNOWN_LINUX_MUSL"; + "i686-unknown-linux-musl" = "I686_UNKNOWN_LINUX_MUSL"; + "arm-unknown-linux-musleabihf" = "ARM_UNKNOWN_LINUX_MUSLEABIHF"; + }; + + pkgsNative = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + pkgs = if target != null then - import pkgsSrc { + import nixpkgs { inherit system; crossSystem = { config = target; isStatic = true; }; - overlays = [ cargo2nixOverlay ]; + overlays = [ (import rust-overlay) ]; } else - import pkgsSrc { - inherit system; - overlays = [ cargo2nixOverlay ]; - }; + pkgsNative; - /* Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. - This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust - to avoid incompatibilities with time_t between different versions of musl - (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. - So we want a Rust release that is bound to our Nix repository to avoid these problems. - See here for more info: https://musl.libc.org/time64.html - Because Cargo2nix does not support the Rust environment shipped by NixOS, - we emulate the structure of the Rust object created by rustOverlay. - In practise, rustOverlay ships rustc+cargo in a single derivation while - NixOS ships them in separate ones. We reunite them with symlinkJoin. - */ - toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl" - || target == "aarch64-unknown-linux-musl" then { - rustVersion = "1.68.0"; - extraRustComponents = [ "clippy" ]; - } else { - rustToolchain = pkgs.symlinkJoin { - name = "rust-static-toolchain-${target}"; - paths = [ - pkgs.rustPlatform.rust.cargo - pkgs.rustPlatform.rust.rustc - # clippy not needed, it only runs on amd64 - ]; - }; - }; + inherit (pkgs) lib stdenv; - buildEnv = (drv: - { - rustc = drv.setBuildEnv; - clippy = '' - ${drv.setBuildEnv or ""} - echo - echo --- BUILDING WITH CLIPPY --- - echo - - export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" - export RUSTC="''${CLIPPY_DRIVER}" - ''; - }.${compiler}); - - /* Cargo2nix provides many overrides by default, you can take inspiration from them: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - - You can have a complete list of the available options by looking at the overriden object, mkcrate: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix - */ - packageOverrides = pkgs: - pkgs.rustBuilder.overrides.all ++ [ - /* [1] We add some logic to compile our crates with clippy, it provides us many additional lints - - [2] We need to alter Nix hardening to make static binaries: PIE, - Position Independent Executables seems to be supported only on amd64. Having - this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. - Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets - (only amd64 curently) through the `-static-pie` flag. - PIE is a feature used by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - - [3] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. - - [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. - However the features to do so get activated for some reason (due to a bug in cargo2nix?), - so disable them manually here. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; - overrideAttrs = drv: - (if git_version != null then { - # [3] - preConfigure = '' - ${drv.preConfigure or ""} - export GIT_VERSION="${git_version}" - ''; - } else - { }) // { - # [1] - setBuildEnv = (buildEnv drv); - # [2] - hardeningDisable = [ "pie" ]; - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_db"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_util"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_table"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_block"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_model"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_api"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_web"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "k2v-client"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "libsodium-sys"; - overrideArgs = old: { - features = [ ]; # [4] - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "zstd-sys"; - overrideArgs = old: { - features = [ ]; # [4] - }; - }) + toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override { + targets = lib.optionals (target != null) [ rustTarget ]; + extensions = [ + "rust-src" + "rustfmt" ]; + }); + + craneLib = (crane.mkLib pkgs).overrideToolchain toolchainFn; + + src = craneLib.cleanCargoSource ../.; /* We ship some parts of the code disabled by default by putting them behind a flag. It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). @@ -189,17 +68,16 @@ let rootFeatures = if features != null then features else - ([ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/k2v" ] ++ (if release then [ - "garage/consul-discovery" - "garage/kubernetes-discovery" - "garage/metrics" - "garage/telemetry-otlp" - "garage/lmdb" - "garage/sqlite" - ] else - [ ])); + ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [ + "consul-discovery" + "kubernetes-discovery" + "metrics" + "telemetry-otlp" + "syslog" + "journald" + ])); - packageFun = import ../Cargo.nix; + featuresStr = lib.concatStringsSep "," rootFeatures; /* We compile fully static binaries with musl to simplify deployment on most systems. When possible, we reactivate PIE hardening (see above). @@ -210,12 +88,9 @@ let For more information on static builds, please refer to Rust's RFC 1721. https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage */ - - codegenOpts = { - "armv6l-unknown-linux-musleabihf" = [ - "target-feature=+crt-static" - "link-arg=-static" - ]; # compile as dynamic with static-pie + codegenOptsMap = { + "x86_64-unknown-linux-musl" = + [ "target-feature=+crt-static" "link-arg=-static-pie" ]; "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" @@ -224,17 +99,106 @@ let "target-feature=+crt-static" "link-arg=-static" ]; # segfault with static-pie - "x86_64-unknown-linux-musl" = - [ "target-feature=+crt-static" "link-arg=-static-pie" ]; + "armv6l-unknown-linux-musleabihf" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # compile as dynamic with static-pie }; - # NixOS and Rust/Cargo triples do not match for ARM, fix it here. - rustTarget = if target == "armv6l-unknown-linux-musleabihf" then - "arm-unknown-linux-musleabihf" - else - target; + codegenOpts = if target != null then codegenOptsMap.${target} else [ + "link-arg=-fuse-ld=mold" + ]; -in pkgs.rustBuilder.makePackageSet ({ - inherit release packageFun packageOverrides codegenOpts rootFeatures; - target = rustTarget; -} // toolchainOptions) + commonArgs = + { + inherit src; + pname = "garage"; + version = "dev"; + + strictDeps = true; + cargoExtraArgs = "--locked --features ${featuresStr}"; + cargoTestExtraArgs = "--workspace"; + + nativeBuildInputs = [ + pkgsNative.protobuf + pkgs.stdenv.cc + ] ++ lib.optionals (target == null) [ + pkgs.clang + pkgs.mold + ]; + + CARGO_PROFILE = if release then "release" else "dev"; + CARGO_BUILD_RUSTFLAGS = + lib.concatStringsSep + " " + (builtins.map (flag: "-C ${flag}") codegenOpts); + } + // + (if rustTarget != null then { + CARGO_BUILD_TARGET = rustTarget; + + "CARGO_TARGET_${rustTargetEnvMap.${rustTarget}}_LINKER" = "${stdenv.cc.targetPrefix}cc"; + + HOST_CC = "${stdenv.cc.nativePrefix}cc"; + TARGET_CC = "${stdenv.cc.targetPrefix}cc"; + } else { + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "clang"; + }); + +in rec { + toolchain = toolchainFn pkgs; + + devShell = pkgs.mkShell { + buildInputs = [ + toolchain + ] ++ (with pkgs; [ + protobuf + clang + mold + ]); + }; + + # ---- building garage ---- + + garage-deps = craneLib.buildDepsOnly commonArgs; + + garage = craneLib.buildPackage (commonArgs // { + cargoArtifacts = garage-deps; + + doCheck = false; + } // + (if git_version != null then { + version = git_version; + GIT_VERSION = git_version; + } else {})); + + # ---- testing garage ---- + + garage-test-bin = craneLib.cargoBuild (commonArgs // { + cargoArtifacts = garage-deps; + + pname = "garage-tests"; + + CARGO_PROFILE = "test"; + cargoExtraArgs = "${commonArgs.cargoExtraArgs} --tests --workspace"; + doCheck = false; + }); + + garage-test = craneLib.cargoTest (commonArgs // { + cargoArtifacts = garage-test-bin; + nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ + pkgs.cacert + ]; + } // extraTestEnv); + + # ---- source code linting ---- + + garage-cargo-fmt = craneLib.cargoFmt (commonArgs // { + cargoExtraArgs = ""; + }); + + garage-cargo-clippy = craneLib.cargoClippy (commonArgs // { + cargoArtifacts = garage-deps; + cargoClippyExtraArgs = "--all-targets -- -D warnings"; + }); +} diff --git a/nix/kaniko.nix b/nix/kaniko.nix deleted file mode 100644 index 8380fa2f..00000000 --- a/nix/kaniko.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "kaniko"; - version = "1.9.2"; - - src = pkgs.fetchFromGitHub { - owner = "GoogleContainerTools"; - repo = "kaniko"; - rev = "v${version}"; - sha256 = "dXQ0/o1qISv+sjNVIpfF85bkbM9sGOGwqVbWZpMWfMY="; - }; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; - homepage = "https://github.com/GoogleContainerTools/kaniko"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix deleted file mode 100644 index 1090a6ef..00000000 --- a/nix/manifest-tool.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "manifest-tool"; - version = "2.0.5"; - - src = pkgs.fetchFromGitHub { - owner = "estesp"; - repo = "manifest-tool"; - rev = "v${version}"; - sha256 = "hjCGKnE0yrlnF/VIzOwcDzmQX3Wft+21KCny/opqdLg="; - } + "/v2"; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "Command line tool to create and query container image manifest list/indexes"; - homepage = "https://github.com/estesp/manifest-tool"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/toolchain.nix b/nix/toolchain.nix deleted file mode 100644 index 532db74e..00000000 --- a/nix/toolchain.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ system ? builtins.currentSystem, }: - -with import ./common.nix; - -let - platforms = [ - #"x86_64-unknown-linux-musl" - "i686-unknown-linux-musl" - #"aarch64-unknown-linux-musl" - "armv6l-unknown-linux-musleabihf" - ]; - pkgsList = builtins.map (target: - import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - }) platforms; - pkgsHost = import pkgsSrc { }; - lib = pkgsHost.lib; - kaniko = (import ./kaniko.nix) pkgsHost; - winscp = (import ./winscp.nix) pkgsHost; - manifestTool = (import ./manifest-tool.nix) pkgsHost; -in lib.flatten (builtins.map (pkgs: [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - pkgs.buildPackages.stdenv.cc -]) pkgsList) ++ [ kaniko winscp manifestTool ] - diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh index 33d739fa..708c2c43 100755 --- a/script/dev-bucket.sh +++ b/script/dev-bucket.sh @@ -9,11 +9,22 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" -garage -c /tmp/config.1.toml bucket create eprouvette -KEY_INFO=$(garage -c /tmp/config.1.toml key create opérateur) +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi + +$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette +if [ "$GARAGE_08" = "1" ]; then + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur) +else + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur) +fi ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'` -garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY +$GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index fa0a950e..998ffdb9 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -11,11 +11,16 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m") export RUST_BACKTRACE=1 -export RUST_LOG=garage=info,garage_api=debug +export RUST_LOG=garage=info,garage_api_common=debug,garage_api_s3=debug MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" -WHICH_GARAGE=$(which garage || exit 1) -echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "${MAIN_LABEL} Found garage at: ${GARAGE_BIN}\n" +else + echo -en "${MAIN_LABEL} Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version NETWORK_SECRET="$(openssl rand -hex 32)" @@ -28,6 +33,7 @@ LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m" cat > $CONF_PATH <&1|while read r; do echo -en "$LABEL $r\n"; done) & +($GARAGE_BIN -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & done # >>>>>>>>>>>>>>>> END FOR LOOP ON NODES @@ -73,14 +79,14 @@ fi sleep 3 # Establish connections between nodes for count in $(seq 1 3); do - NODE=$(garage -c /tmp/config.$count.toml node id -q) + NODE=$($GARAGE_BIN -c /tmp/config.$count.toml node id -q) for count2 in $(seq 1 3); do - garage -c /tmp/config.$count2.toml node connect $NODE + $GARAGE_BIN -c /tmp/config.$count2.toml node connect $NODE done done RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo -en "${MAIN_LABEL} Garage did not start" diff --git a/script/dev-configure.sh b/script/dev-configure.sh index 9c24bf4b..0649cdbe 100755 --- a/script/dev-configure.sh +++ b/script/dev-configure.sh @@ -9,9 +9,17 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version + sleep 5 RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo "garage did not start in time, failing." @@ -21,12 +29,20 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do sleep 1 done -garage -c /tmp/config.1.toml status \ - | grep 'NO ROLE' \ - | grep -Po '^[0-9a-f]+' \ - | while read id; do - garage -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G - done +if [ "$GARAGE_08" = "1" ]; then + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1 + done +else + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G + done +fi -garage -c /tmp/config.1.toml layout config -r 1 -garage -c /tmp/config.1.toml layout apply --version 1 +$GARAGE_BIN -c /tmp/config.1.toml layout apply --version 1 diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh index 9436c2c7..41f1fdde 100644 --- a/script/dev-env-aws.sh +++ b/script/dev-env-aws.sh @@ -1,7 +1,8 @@ export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` export AWS_DEFAULT_REGION='garage' - +export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' +# FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0. function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } aws --version diff --git a/script/helm/README.md b/script/helm/README.md index 5f919a23..1b22e604 100644 --- a/script/helm/README.md +++ b/script/helm/README.md @@ -1,3 +1,3 @@ # Garage helm3 chart -Documentation is located [here](/doc/book/cookbook/kubernetes.md). +Documentation is located [here](https://garagehq.deuxfleurs.fr/documentation/cookbook/kubernetes/). diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 31c5d4e2..b3a7b921 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -1,24 +1,18 @@ apiVersion: v2 name: garage description: S3-compatible object store for small self-hosted geo-distributed deployments - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application +version: 0.7.3 +appVersion: "v1.3.1" +home: https://garagehq.deuxfleurs.fr/ +icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.1 +keywords: +- geo-distributed +- read-after-write-consistency +- s3-compatible -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "v0.8.4" +sources: +- https://git.deuxfleurs.fr/Deuxfleurs/garage.git + +maintainers: [] diff --git a/script/helm/garage/README.md b/script/helm/garage/README.md new file mode 100644 index 00000000..bdf69ec4 --- /dev/null +++ b/script/helm/garage/README.md @@ -0,0 +1,95 @@ +# garage + +![Version: 0.7.3](https://img.shields.io/badge/Version-0.7.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.1](https://img.shields.io/badge/AppVersion-v1.3.1-informational?style=flat-square) + +S3-compatible object store for small self-hosted geo-distributed deployments + +**Homepage:** + +## Source Code + +* + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet | +| deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) | +| deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start | +| environment | object | `{}` | | +| extraVolumeMounts | object | `{}` | | +| extraVolumes | object | `{}` | | +| fullnameOverride | string | `""` | | +| garage.blockSize | string | `"1048576"` | Defaults is 1MB An increase can result in better performance in certain scenarios https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size | +| garage.bootstrapPeers | list | `[]` | This is not required if you use the integrated kubernetes discovery | +| garage.compressionLevel | string | `"1"` | zstd compression level of stored blocks https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level | +| garage.dbEngine | string | `"lmdb"` | Can be changed for better performance on certain systems https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0 | +| garage.existingConfigMap | string | `""` | if not empty string, allow using an existing ConfigMap for the garage.toml, if set, ignores garage.toml | +| garage.garageTomlString | string | `""` | String Template for the garage configuration if set, ignores above values. Values can be templated, see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ | +| garage.kubernetesSkipCrd | bool | `false` | Set to true if you want to use k8s discovery but install the CRDs manually outside of the helm chart, for example if you operate at namespace level without cluster ressources | +| garage.metadataAutoSnapshotInterval | string | `""` | If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval | +| garage.replicationMode | string | `"3"` | Default to 3 replicas, see the replication_mode section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode | +| garage.rpcBindAddr | string | `"[::]:3901"` | | +| garage.rpcSecret | string | `""` | If not given, a random secret will be generated and stored in a Secret object | +| garage.s3.api.region | string | `"garage"` | | +| garage.s3.api.rootDomain | string | `".s3.garage.tld"` | | +| garage.s3.web.index | string | `"index.html"` | | +| garage.s3.web.rootDomain | string | `".web.garage.tld"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"dxflrs/amd64_garage"` | default to amd64 docker image | +| image.tag | string | `""` | set the image tag, please prefer using the chart version and not this to avoid compatibility issues | +| imagePullSecrets | list | `[]` | set if you need credentials to pull your custom image | +| ingress.s3.api.annotations | object | `{}` | Rely _either_ on the className or the annotation below but not both! If you want to use the className, set className: "nginx" and replace "nginx" by an Ingress controller name, examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). | +| ingress.s3.api.enabled | bool | `false` | | +| ingress.s3.api.hosts[0] | object | `{"host":"s3.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | garage S3 API endpoint, to be used with awscli for example | +| ingress.s3.api.hosts[1] | object | `{"host":"*.s3.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | garage S3 API endpoint, DNS style bucket access | +| ingress.s3.api.labels | object | `{}` | | +| ingress.s3.api.tls | list | `[]` | | +| ingress.s3.web.annotations | object | `{}` | Rely _either_ on the className or the annotation below but not both! If you want to use the className, set className: "nginx" and replace "nginx" by an Ingress controller name, examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). | +| ingress.s3.web.enabled | bool | `false` | | +| ingress.s3.web.hosts[0] | object | `{"host":"*.web.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | wildcard website access with bucket name prefix | +| ingress.s3.web.hosts[1] | object | `{"host":"mywebpage.example.com","paths":[{"path":"/","pathType":"Prefix"}]}` | specific bucket access with FQDN bucket | +| ingress.s3.web.labels | object | `{}` | | +| ingress.s3.web.tls | list | `[]` | | +| initImage.pullPolicy | string | `"IfNotPresent"` | | +| initImage.repository | string | `"busybox"` | | +| initImage.tag | string | `"stable"` | | +| livenessProbe | object | `{}` | Specifies a livenessProbe | +| monitoring.metrics.enabled | bool | `false` | If true, a service for monitoring is created with a prometheus.io/scrape annotation | +| monitoring.metrics.serviceMonitor.enabled | bool | `false` | If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator | +| monitoring.metrics.serviceMonitor.interval | string | `"15s"` | | +| monitoring.metrics.serviceMonitor.labels | object | `{}` | | +| monitoring.metrics.serviceMonitor.path | string | `"/metrics"` | | +| monitoring.metrics.serviceMonitor.relabelings | list | `[]` | | +| monitoring.metrics.serviceMonitor.scheme | string | `"http"` | | +| monitoring.metrics.serviceMonitor.scrapeTimeout | string | `"10s"` | | +| monitoring.metrics.serviceMonitor.tlsConfig | object | `{}` | | +| monitoring.tracing.sink | string | `""` | specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317` | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| persistence.data.hostPath | string | `"/var/lib/garage/data"` | | +| persistence.data.size | string | `"100Mi"` | | +| persistence.enabled | bool | `true` | | +| persistence.meta.hostPath | string | `"/var/lib/garage/meta"` | | +| persistence.meta.size | string | `"100Mi"` | | +| podAnnotations | object | `{}` | additonal pod annotations | +| podSecurityContext.fsGroup | int | `1000` | | +| podSecurityContext.runAsGroup | int | `1000` | | +| podSecurityContext.runAsNonRoot | bool | `true` | | +| podSecurityContext.runAsUser | int | `1000` | | +| readinessProbe | object | `{}` | Specifies a readinessProbe | +| resources | object | `{}` | | +| securityContext.capabilities | object | `{"drop":["ALL"]}` | The default security context is heavily restricted, feel free to tune it to your requirements | +| securityContext.readOnlyRootFilesystem | bool | `true` | | +| service.s3.api.port | int | `3900` | | +| service.s3.web.port | int | `3902` | | +| service.type | string | `"ClusterIP"` | You can rely on any service to expose your cluster - ClusterIP (+ Ingress) - NodePort (+ Ingress) - LoadBalancer | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml index 5cc7a45e..ab5b84db 100644 --- a/script/helm/garage/templates/configmap.yaml +++ b/script/helm/garage/templates/configmap.yaml @@ -1,7 +1,53 @@ +{{- if not .Values.garage.existingConfigMap }} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "garage.fullname" . }}-config data: garage.toml: |- - {{- tpl (index (index .Values.garage) "garage.toml") $ | nindent 4 }} + {{- if .Values.garage.garageTomlString }} + {{- tpl (index (index .Values.garage) "garageTomlString") $ | nindent 4 }} + {{- else }} + metadata_dir = "/mnt/meta" + data_dir = "/mnt/data" + + db_engine = "{{ .Values.garage.dbEngine }}" + + block_size = {{ .Values.garage.blockSize }} + + replication_mode = "{{ .Values.garage.replicationMode }}" + + compression_level = {{ .Values.garage.compressionLevel }} + + {{- if .Values.garage.metadataAutoSnapshotInterval }} + metadata_auto_snapshot_interval = {{ .Values.garage.metadataAutoSnapshotInterval | quote }} + {{- end }} + + rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" + # rpc_secret will be populated by the init container from a k8s secret object + rpc_secret = "__RPC_SECRET_REPLACE__" + + bootstrap_peers = {{ .Values.garage.bootstrapPeers }} + + [kubernetes_discovery] + namespace = "{{ .Release.Namespace }}" + service_name = "{{ include "garage.fullname" . }}" + skip_crd = {{ .Values.garage.kubernetesSkipCrd }} + + [s3_api] + s3_region = "{{ .Values.garage.s3.api.region }}" + api_bind_addr = "[::]:3900" + root_domain = "{{ .Values.garage.s3.api.rootDomain }}" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = "{{ .Values.garage.s3.web.rootDomain }}" + index = "{{ .Values.garage.s3.web.index }}" + + [admin] + api_bind_addr = "[::]:3903" + {{- if .Values.monitoring.tracing.sink }} + trace_sink = "{{ .Values.monitoring.tracing.sink }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/script/helm/garage/templates/service-headless.yaml b/script/helm/garage/templates/service-headless.yaml new file mode 100644 index 00000000..7bc9f2cc --- /dev/null +++ b/script/helm/garage/templates/service-headless.yaml @@ -0,0 +1,22 @@ +{{- if eq .Values.deployment.kind "StatefulSet" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "garage.fullname" . }}-headless + labels: + {{- include "garage.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ .Values.service.s3.api.port }} + targetPort: 3900 + protocol: TCP + name: s3-api + - port: {{ .Values.service.s3.web.port }} + targetPort: 3902 + protocol: TCP + name: s3-web + selector: + {{- include "garage.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/script/helm/garage/templates/service.yaml b/script/helm/garage/templates/service.yaml index 37218872..887c90d0 100644 --- a/script/helm/garage/templates/service.yaml +++ b/script/helm/garage/templates/service.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "garage.fullname" . }} labels: {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.service.type }} ports: @@ -37,4 +41,4 @@ spec: name: metrics selector: {{- include "garage.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/script/helm/garage/templates/workload.yaml b/script/helm/garage/templates/workload.yaml index 340c0054..d144cb41 100644 --- a/script/helm/garage/templates/workload.yaml +++ b/script/helm/garage/templates/workload.yaml @@ -10,11 +10,11 @@ spec: {{- include "garage.selectorLabels" . | nindent 6 }} {{- if eq .Values.deployment.kind "StatefulSet" }} replicas: {{ .Values.deployment.replicaCount }} - serviceName: {{ include "garage.fullname" . }} + serviceName: {{ include "garage.fullname" . }}-headless + podManagementPolicy: {{ .Values.deployment.podManagementPolicy }} {{- end }} template: metadata: - annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} @@ -63,6 +63,10 @@ spec: name: web-api - containerPort: 3903 name: admin + {{- with .Values.environment }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: meta mountPath: /mnt/meta @@ -71,15 +75,17 @@ spec: - name: etc mountPath: /etc/garage.toml subPath: garage.toml - # TODO - # livenessProbe: - # httpGet: - # path: / - # port: 3900 - # readinessProbe: - # httpGet: - # path: / - # port: 3900 + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} volumes: @@ -105,6 +111,9 @@ spec: - name: data emptyDir: {} {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index 02a6651b..5e419fe2 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -4,33 +4,34 @@ # Garage configuration. These values go to garage.toml garage: - # Can be changed for better performance on certain systems + # -- Can be changed for better performance on certain systems # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0 - dbEngine: "sled" + dbEngine: "lmdb" - # Defaults is 1MB + # -- Defaults is 1MB # An increase can result in better performance in certain scenarios # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size blockSize: "1048576" - # Tuning parameters for the sled DB engine - # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#sled-cache-capacity - sledCacheCapacity: "134217728" - sledFlushEveryMs: "2000" - - # Default to 3 replicas, see the replication_mode section at + # -- Default to 3 replicas, see the replication_mode section at # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode replicationMode: "3" - # zstd compression level of stored blocks + # -- zstd compression level of stored blocks # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level compressionLevel: "1" + # -- If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval + metadataAutoSnapshotInterval: "" + rpcBindAddr: "[::]:3901" - # If not given, a random secret will be generated and stored in a Secret object + # -- If not given, a random secret will be generated and stored in a Secret object rpcSecret: "" - # This is not required if you use the integrated kubernetes discovery + # -- This is not required if you use the integrated kubernetes discovery bootstrapPeers: [] + # -- Set to true if you want to use k8s discovery but install the CRDs manually outside + # of the helm chart, for example if you operate at namespace level without cluster ressources kubernetesSkipCrd: false s3: api: @@ -39,52 +40,16 @@ garage: web: rootDomain: ".web.garage.tld" index: "index.html" - # Template for the garage configuration - # Values can be templated - # ref: https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ - garage.toml: |- - metadata_dir = "/mnt/meta" - data_dir = "/mnt/data" - db_engine = "{{ .Values.garage.dbEngine }}" + # -- if not empty string, allow using an existing ConfigMap for the garage.toml, + # if set, ignores garage.toml + existingConfigMap: "" - block_size = {{ .Values.garage.blockSize }} - - {{- if eq .Values.garage.dbEngine "sled"}} - sled_cache_capacity = {{ .Values.garage.sledCacheCapacity }} - sled_flush_every_ms = {{ .Values.garage.sledFlushEveryMs }} - {{- end }} - - replication_mode = "{{ .Values.garage.replicationMode }}" - - compression_level = {{ .Values.garage.compressionLevel }} - - rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" - # rpc_secret will be populated by the init container from a k8s secret object - rpc_secret = "__RPC_SECRET_REPLACE__" - - bootstrap_peers = {{ .Values.garage.bootstrapPeers }} - - [kubernetes_discovery] - namespace = "{{ .Release.Namespace }}" - service_name = "{{ include "garage.fullname" . }}" - skip_crd = {{ .Values.garage.kubernetesSkipCrd }} - - [s3_api] - s3_region = "{{ .Values.garage.s3.api.region }}" - api_bind_addr = "[::]:3900" - root_domain = "{{ .Values.garage.s3.api.rootDomain }}" - - [s3_web] - bind_addr = "[::]:3902" - root_domain = "{{ .Values.garage.s3.web.rootDomain }}" - index = "{{ .Values.garage.s3.web.index }}" - - [admin] - api_bind_addr = "[::]:3903" - {{- if .Values.monitoring.tracing.sink }} - trace_sink = "{{ .Values.monitoring.tracing.sink }}" - {{- end }} + # -- String Template for the garage configuration + # if set, ignores above values. + # Values can be templated, + # see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garageTomlString: "" # Data persistence persistence: @@ -102,14 +67,18 @@ persistence: # Deployment configuration deployment: - # Switchable to DaemonSet + # -- Switchable to DaemonSet kind: StatefulSet - # Number of StatefulSet replicas/garage nodes to start + # -- Number of StatefulSet replicas/garage nodes to start replicaCount: 3 + # -- If using statefulset, allow Parallel or OrderedReady (default) + podManagementPolicy: OrderedReady image: + # -- default to amd64 docker image repository: dxflrs/amd64_garage - # please prefer using the chart version and not this tag + # -- set the image tag, please prefer using the chart version and not this + # to avoid compatibility issues tag: "" pullPolicy: IfNotPresent @@ -118,19 +87,21 @@ initImage: tag: stable pullPolicy: IfNotPresent +# -- set if you need credentials to pull your custom image imagePullSecrets: [] nameOverride: "" fullnameOverride: "" serviceAccount: - # Specifies whether a service account should be created + # -- Specifies whether a service account should be created create: true - # Annotations to add to the service account + # -- Annotations to add to the service account annotations: {} - # The name of the service account to use. + # -- The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" +# -- additonal pod annotations podAnnotations: {} podSecurityContext: @@ -140,7 +111,7 @@ podSecurityContext: runAsNonRoot: true securityContext: - # The default security context is heavily restricted + # -- The default security context is heavily restricted, # feel free to tune it to your requirements capabilities: drop: @@ -148,11 +119,13 @@ securityContext: readOnlyRootFilesystem: true service: - # You can rely on any service to expose your cluster + # -- You can rely on any service to expose your cluster # - ClusterIP (+ Ingress) # - NodePort (+ Ingress) # - LoadBalancer type: ClusterIP + # -- Annotations to add to the service + annotations: {} s3: api: port: 3900 @@ -164,20 +137,23 @@ ingress: s3: api: enabled: false - # Rely either on the className or the annotation below but not both - # replace "nginx" by an Ingress controller - # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers + # -- Rely _either_ on the className or the annotation below but not both! + # If you want to use the className, set # className: "nginx" + # and replace "nginx" by an Ingress controller name, + # examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). annotations: {} # kubernetes.io/ingress.class: "nginx" # kubernetes.io/tls-acme: "true" labels: {} hosts: - - host: "s3.garage.tld" # garage S3 API endpoint + # -- garage S3 API endpoint, to be used with awscli for example + - host: "s3.garage.tld" paths: - path: / pathType: Prefix - - host: "*.s3.garage.tld" # garage S3 API endpoint, DNS style bucket access + # -- garage S3 API endpoint, DNS style bucket access + - host: "*.s3.garage.tld" paths: - path: / pathType: Prefix @@ -187,20 +163,23 @@ ingress: # - kubernetes.docker.internal web: enabled: false - # Rely either on the className or the annotation below but not both - # replace "nginx" by an Ingress controller - # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers + # -- Rely _either_ on the className or the annotation below but not both! + # If you want to use the className, set # className: "nginx" + # and replace "nginx" by an Ingress controller name, + # examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" labels: {} hosts: - - host: "*.web.garage.tld" # wildcard website access with bucket name prefix + # -- wildcard website access with bucket name prefix + - host: "*.web.garage.tld" paths: - path: / pathType: Prefix - - host: "mywebpage.example.com" # specific bucket access with FQDN bucket + # -- specific bucket access with FQDN bucket + - host: "mywebpage.example.com" paths: - path: / pathType: Prefix @@ -218,18 +197,39 @@ resources: {} # cpu: 100m # memory: 512Mi +# -- Specifies a livenessProbe +livenessProbe: {} + #httpGet: + # path: /health + # port: 3903 + #initialDelaySeconds: 5 + #periodSeconds: 30 +# -- Specifies a readinessProbe +readinessProbe: {} + #httpGet: + # path: /health + # port: 3903 + #initialDelaySeconds: 5 + #periodSeconds: 30 + nodeSelector: {} tolerations: [] affinity: {} +environment: {} + +extraVolumes: {} + +extraVolumeMounts: {} + monitoring: metrics: - # If true, a service for monitoring is created with a prometheus.io/scrape annotation + # -- If true, a service for monitoring is created with a prometheus.io/scrape annotation enabled: false serviceMonitor: - # If true, a ServiceMonitor CRD is created for a prometheus operator + # -- If true, a ServiceMonitor CRD is created for a prometheus operator # https://github.com/coreos/prometheus-operator enabled: false path: /metrics @@ -241,4 +241,5 @@ monitoring: scrapeTimeout: 10s relabelings: [] tracing: + # -- specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317` sink: "" diff --git a/script/jepsen.garage/.envrc b/script/jepsen.garage/.envrc new file mode 100644 index 00000000..1d953f4b --- /dev/null +++ b/script/jepsen.garage/.envrc @@ -0,0 +1 @@ +use nix diff --git a/script/jepsen.garage/.gitignore b/script/jepsen.garage/.gitignore new file mode 100644 index 00000000..31842a96 --- /dev/null +++ b/script/jepsen.garage/.gitignore @@ -0,0 +1,17 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +/.prepl-port +.hgignore +.hg/ +.direnv +/store +/store.* +.vagrant diff --git a/script/jepsen.garage/README.md b/script/jepsen.garage/README.md new file mode 100644 index 00000000..50c7eb38 --- /dev/null +++ b/script/jepsen.garage/README.md @@ -0,0 +1,166 @@ +# jepsen.garage + +Jepsen checking of Garage consistency properties. + +## Usage + +Requirements: + +- vagrant +- VirtualBox, configured so that nodes can take an IP in a private network `192.168.56.0/24` (it's the default) +- a user that can create VirtualBox VMs +- leiningen +- gnuplot + +Set up VMs before running tests: + +``` +vagrant up +``` + +Run tests: see commands below. + + +## Results + +### Register linear, without timestamp patch + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 20 --workload reg1 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: without the timestamp patch, nodes will create objects using their + local clock only as a timestamp, so the ordering will be all over the place if + clocks are scrambled. + +Results with timestamp patch (`--patch tsfix2`): + +- No failure with clock-scramble nemesis + +- Fails with clock-scramble nemesis + partition nemesis (`--scenario cp`). + +**This test is expected to fail.** +Indeed, S3 objects are not meant to behave like linearizable registers. +TODO explain using a counter-example + + +### Read-after-write CRDT register model + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload reg2 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: old values are not overwritten correctly when their timestamps are in the future. + +Results with timestamp patch (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + partition nemesis (`--scenario cp`). + This proves that `tsfix2` (PR#543) does improve consistency. + +- **Fails with layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage reg2/20231024T120806.899+0200`. + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +### Set, basic test (write some items, then read) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 200 --concurrency 200 --workload set1 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run + +- Does not seem to fail with only the layout reconfiguation nemesis (<10 runs), although theoretically it could + +- **Fails with the partition + layout reconfiguration nemesis** (`--scenario pr`). + Example of a failed run: `garage set1/20231024T172214.488+0200` (1 failure in 4 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- The tests are buggy and often result in an "unknown" validity status, which + is caused by some requests not returning results during network partitions or + other nemesis-induced broken cluster states. However, when the tests were + able to finish, there were no failures with scenarios `r`, `pr`, `cpr`, + `dpr`. + + +### Set, continuous test (interspersed reads and writes) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload set2 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + db nemesis + partition nemesis (`--scenario cdp`) (0 failures in 10 runs). + +- **Fails with just layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage set2/20231025T141940.198+0200` (10 failures in 10 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +## NLnet task 3 final results + +- With code from task3 (`--patch task3c`): [reg2 and set2](results/Results-2023-12-13-task3c.png), [set1](results/Results-2023-12-14-task3-set1.png). +- Without (`--patch tsfix2`): [reg2 and set2](results/Results-2023-12-13-tsfix2.png), set1 TBD. + +## Investigating (and fixing) errors + +### Segfaults + +They are due to the download being interrupted in the middle (^C during first launch on clean VMs), the `garage` binary is truncated. +Add `:force?` to the `cached-wget!` call in `daemon.clj` to re-download the binary, +or restar the VMs to clear temporary files. + +### In `jepsen.garage`: prefix wierdness + +In `store/garage set1/20231019T163358.615+0200`: + +``` +INFO [2023-10-19 16:35:20,977] clojure-agent-send-off-pool-207 - jepsen.garage.set list results for prefix set20/ : (set13/0 set13/1 set13/10 set13/11 set13/12 set13/13 set13/14 set13/15 set13/16 set13/17 set13/18 set13/19 set13/2 set13/20 set13/21 set13/22 set13/23 set13/24 set13/25 set13/26 set13/27 set13/28 set13/29 set13/3 set13/30 set13/31 set13/32 set13/33 set13/34 set13/35 set13/36 set13/37 set13/38 set13/39 set13/4 set13/40 set13/41 set13/42 set13/43 set13/44 set13/45 set13/46 set13/47 set13/48 set13/49 set13/5 set13/50 set13/51 set13/52 set13/53 set13/54 set13/55 set13/56 set13/57 set13/58 set13/59 set13/6 set13/60 set13/61 set13/62 set13/63 set13/64 set13/65 set13/66 set13/67 set13/68 set13/69 set13/7 set13/70 set13/71 set13/72 set13/73 set13/74 set13/75 set13/76 set13/77 set13/78 set13/79 set13/8 set13/80 set13/81 set13/82 set13/83 set13/84 set13/85 set13/86 set13/87 set13/88 set13/89 set13/9 set13/90 set13/91 set13/92 set13/93 set13/94 set13/95 set13/96 set13/97 set13/98 set13/99) (node: http://192.168.56.25:3900 ) +``` + +After inspecting, the actual S3 call made was with prefix "set13/", so at least this is not an error in Garage itself but in the jepsen code. + +Finally found out that this was due to closures not correctly capturing their context in the list function in s3api.clj (wtf clojure?) +Not sure exactly where it came from but it seems to have been fixed by making list-inner a separate function and not a sub-function, +and passing all values that were previously in the context (creds and prefix) as additional arguments. + +### `reg2` test inconsistency, even with timestamp fix + +The reg2 test is our custom checker for CRDT read-after-write on individual object keys, acting as registers which can be updated. +The test fails without the timestamp fix, which is expected as the clock scrambler will prevent nodes from having a correct ordering of objects. + +With the timestamp fix (`--patch tsfix1`), the happenned-before relationship should at least be respected, meaning that when a PutObject call starts +after another PutObject call has ended, the second call should overwrite the value of the first call, and that value should not be +readable by future GetObject calls. +However, we observed inconsistencies even with the timestamp fix. + +The inconsistencies seemed to always happenned after writing a nil value, which translates to a DeleteObject call +instead of a PutObject. By removing the possibility of writing nil values, therefore only doing +PutObject calls, the issue disappears. There is therefore an issue to fix in DeleteObject. + +The issue in DeleteObject seems to have been fixed by commit `c82d91c6bccf307186332b6c5c6fc0b128b1b2b1`, which can be used using `--patch tsfix2`. + + +## License + +Copyright © 2023 Alex Auvolat + +This program and the accompanying materials are made available under the +terms of the GNU Affero General Public License v3.0. diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile new file mode 100644 index 00000000..2ac24f98 --- /dev/null +++ b/script/jepsen.garage/Vagrantfile @@ -0,0 +1,40 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# + +def vm(config, hostname, ip) + config.vm.hostname = hostname + config.vm.network "private_network", ip: ip +end + +Vagrant.configure("2") do |config| + config.vm.box = "generic/debian10" + + config.vm.provider "virtualbox" do |vb| + vb.gui = false + vb.memory = "512" + vb.customize ["modifyvm", :id, "--vram=12"] + end + + config.vm.provision "shell", inline: <<-SHELL + echo "root:root" | chpasswd + mkdir -p /root/.ssh + echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys + SHELL + + config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end + config.vm.define "n2" do |config| vm(config, "n2", "192.168.56.22") end + config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end + config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end + config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end + config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end + config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end + + #config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end + #config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end + #config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end + #config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end + #config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end + #config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end + #config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end +end diff --git a/script/jepsen.garage/all_tests_1.sh b/script/jepsen.garage/all_tests_1.sh new file mode 100755 index 00000000..f9770865 --- /dev/null +++ b/script/jepsen.garage/all_tests_1.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c task3a tsfix2; do +for ppatch in v093 v1rc1; do + #for psc in c cp cdp r pr cpr dpr; do + for ptsk in reg2 set2; do + for psc in c cp cdp r pr cpr dpr; do + for irun in $(seq 10); do + lein run test --nodes-file nodes.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/all_tests_2.sh b/script/jepsen.garage/all_tests_2.sh new file mode 100755 index 00000000..641643ed --- /dev/null +++ b/script/jepsen.garage/all_tests_2.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c tsfix2; do +for ppatch in tsfix2; do + for psc in cdp r pr cpr dpr; do + for ptsk in set1; do + for irun in $(seq 10); do + lein run test --nodes-file nodes2.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/jaeger.sh b/script/jepsen.garage/jaeger.sh new file mode 100644 index 00000000..7f67b61b --- /dev/null +++ b/script/jepsen.garage/jaeger.sh @@ -0,0 +1,13 @@ +docker stop jaeger +docker rm jaeger + +# UI is on localhost:16686 +# otel-grpc collector is on localhost:4317 +# otel-http collector is on localhost:4318 + +docker run -d --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 4317:4317 \ + -p 4318:4318 \ + -p 16686:16686 \ + jaegertracing/all-in-one:1.50 diff --git a/script/jepsen.garage/nodes.vagrant b/script/jepsen.garage/nodes.vagrant new file mode 100644 index 00000000..9e5694e6 --- /dev/null +++ b/script/jepsen.garage/nodes.vagrant @@ -0,0 +1,7 @@ +192.168.56.21 +192.168.56.22 +192.168.56.23 +192.168.56.24 +192.168.56.25 +192.168.56.26 +192.168.56.27 diff --git a/script/jepsen.garage/nodes2.vagrant b/script/jepsen.garage/nodes2.vagrant new file mode 100644 index 00000000..842bf276 --- /dev/null +++ b/script/jepsen.garage/nodes2.vagrant @@ -0,0 +1,7 @@ +192.168.56.28 +192.168.56.29 +192.168.56.30 +192.168.56.31 +192.168.56.32 +192.168.56.33 +192.168.56.34 diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj new file mode 100644 index 00000000..59d45484 --- /dev/null +++ b/script/jepsen.garage/project.clj @@ -0,0 +1,10 @@ +(defproject jepsen.garage "0.1.0-SNAPSHOT" + :description "Jepsen testing for Garage" + :url "https://git.deuxfleurs.fr/Deuxfleurs/garage" + :license {:name "AGPLv3" + :url "https://www.gnu.org/licenses/agpl-3.0.en.html"} + :main jepsen.garage + :dependencies [[org.clojure/clojure "1.11.1"] + [jepsen "0.3.3-SNAPSHOT"] + [amazonica "0.3.163"]] + :repl-options {:init-ns jepsen.garage}) diff --git a/script/jepsen.garage/results/Results-2023-11-16.png b/script/jepsen.garage/results/Results-2023-11-16.png new file mode 100644 index 00000000..26dac833 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-11-16.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-13-task3c.png b/script/jepsen.garage/results/Results-2023-12-13-task3c.png new file mode 100644 index 00000000..216043c3 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-13-task3c.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png new file mode 100644 index 00000000..147d25e9 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png new file mode 100644 index 00000000..dbff3a95 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png differ diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix new file mode 100644 index 00000000..01e4c845 --- /dev/null +++ b/script/jepsen.garage/shell.nix @@ -0,0 +1,18 @@ +{ pkgs ? import { + overlays = [ + (self: super: { + jdk = super.jdk11; + jre = super.jre11; + }) + ]; +} }: +pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + leiningen + jdk + jna + vagrant + gnuplot + graphviz + ]; +} diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj new file mode 100644 index 00000000..053ad8b6 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -0,0 +1,107 @@ +(ns jepsen.garage + (:require + [clojure.string :as str] + [jepsen + [checker :as checker] + [cli :as cli] + [generator :as gen] + [nemesis :as nemesis] + [tests :as tests]] + [jepsen.os.debian :as debian] + [jepsen.garage + [daemon :as grg] + [nemesis :as grgNemesis] + [reg :as reg] + [set :as set]])) + +(def workloads + "A map of workload names to functions that construct workloads, given opts." + {"reg1" reg/workload1 + "reg2" reg/workload2 + "set1" set/workload1 + "set2" set/workload2}) + +(def scenari + "A map of scenari to the associated nemesis" + {"c" grgNemesis/scenario-c + "cp" grgNemesis/scenario-cp + "r" grgNemesis/scenario-r + "pr" grgNemesis/scenario-pr + "cpr" grgNemesis/scenario-cpr + "cdp" grgNemesis/scenario-cdp + "dpr" grgNemesis/scenario-dpr}) + +(def patches + "A map of patch names to Garage builds" + {"default" "v0.9.0" + "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09" + "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1" + "task3a" "707442f5de416fdbed4681a33b739f0a787b7834" + "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997" + "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b" + "v093" "v0.9.3" + "v1rc1" "v1.0.0-rc1"}) + +(def cli-opts + "Additional command line options." + [["-p" "--patch NAME" "Garage patch to use" + :default "default" + :validate [patches (cli/one-of patches)]] + ["-s" "--scenario NAME" "Nemesis scenario to run" + :default "cp" + :validate [scenari (cli/one-of scenari)]] + ["-r" "--rate HZ" "Approximate number of requests per second, per thread." + :default 10 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--ops-per-key NUM" "Maximum number of operations on any given key." + :default 100 + :parse-fn parse-long + :validate [pos? "Must be a positive integer."]] + ["-w" "--workload NAME" "Workload of test to run" + :default "reg1" + :validate [workloads (cli/one-of workloads)]]]) + +(defn garage-test + "Given an options map from the command line runner (e.g. :nodes, :ssh, + :concurrency, ...), constructs a test map." + [opts] + (let [garage-version (get patches (:patch opts)) + db (grg/db garage-version) + workload ((get workloads (:workload opts)) opts) + scenario ((get scenari (:scenario opts)) (assoc opts :db db))] + (merge tests/noop-test + opts + {:pure-generators true + :name (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts))) + :os debian/os + :db db + :client (:client workload) + :generator (gen/phases + (->> + (:generator workload) + (gen/stagger (/ (:rate opts))) + (gen/nemesis (:generator scenario)) + (gen/time-limit (:time-limit opts))) + (gen/log "Healing cluster") + (gen/nemesis (:final-generator scenario)) + (gen/log "Waiting for recovery") + (gen/sleep 10) + (gen/log "Running final generator") + (gen/clients (:final-generator workload)) + (gen/log "Generators all done")) + :nemesis (:nemesis scenario) + :checker (checker/compose + {:perf (checker/perf (:perf scenario)) + :workload (:checker workload)}) + }))) + + +(defn -main + "Handles command line arguments. Can either run a test, or a web server for + browsing results." + [& args] + (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj new file mode 100644 index 00000000..0ea773fb --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -0,0 +1,152 @@ +(ns jepsen.garage.daemon + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [db :as db]] + [jepsen.control.util :as cu])) + +; CONSTANTS -- HOW GARAGE IS SET UP + +(def base-dir "/opt/garage") +(def data-dir (str base-dir "/data")) +(def meta-dir (str base-dir "/meta")) +(def binary (str base-dir "/garage")) +(def logfile (str base-dir "/garage.log")) +(def pidfile (str base-dir "/garage.pid")) + +(def admin-token "icanhazadmin") +(def access-key-id "GK8bfb6a51286071c6c9cd8bc3") +(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3") +(def bucket-name "jepsen") + +; THE GARAGE DB + +(defn install! + "Download and install Garage" + [node version] + (c/su + (c/trace + (info node "installing garage" version) + (c/exec :mkdir :-p base-dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/cached-wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary)))) + +(defn configure! + "Configure Garage" + [node] + (c/su + (c/trace + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" data-dir "\"\n" + "metadata_dir = \"" meta-dir "\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" admin-token "\"\n" + "trace_sink = \"http://192.168.56.1:4317\"\n") + "/etc/garage.toml")))) + +(defn connect-node! + "Connect a Garage node to the rest of the cluster" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id))))) + +(defn configure-node! + "Configure a Garage node to be part of a cluster layout" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (c/on (jepsen/primary test) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) + +(defn finalize-config! + "Apply the layout and create a key/bucket pair in the cluster" + [node] + (c/trace + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :import access-key-id secret-access-key :--yes) + (c/exec binary :bucket :create bucket-name) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id) + (info node "key info: " (c/exec binary :key :info access-key-id)))) + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (install! node version) + (configure! node) + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server) + (c/exec :sleep 3) + + (jepsen/synchronize test) + (connect-node! test node) + + (jepsen/synchronize test) + (configure-node! test node) + + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (finalize-config! node))) + + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf logfile) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf meta-dir))) + + db/Pause + (pause! [_ test node] + (cu/grepkill! :stop binary)) + (resume! [_ test node] + (cu/grepkill! :cont binary)) + + db/Kill + (kill! [_ test node] + (cu/stop-daemon! binary pidfile)) + (start! [_ test node] + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server)) + + db/LogFiles + (log-files [_ test node] + [logfile]))) + +(defn creds + "Obtain Garage credentials for node" + [node] + {:access-key access-key-id + :secret-key secret-access-key + :endpoint (str "http://" node ":3900") + :bucket bucket-name + :client-config {:path-style-access-enabled true}}) + diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj new file mode 100644 index 00000000..dfce0255 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -0,0 +1,142 @@ +(ns jepsen.garage.nemesis + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [generator :as gen] + [nemesis :as nemesis]] + [jepsen.nemesis.combined :as combined] + [jepsen.garage.daemon :as grg] + [jepsen.control.util :as cu])) + +; ---- reconfiguration nemesis ---- + +(defn configure-present! + "Configure node to be active in new cluster layout" + [test nodes] + (info "configure-present!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs))))) + +(defn configure-absent! + "Configure nodes to be active in new cluster layout" + [test nodes] + (info "configure-absent!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs))))) + +(defn finalize-config! + "Apply the proposed cluster layout" + [test] + (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) + [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] + (if layout-next-version + (do + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version))) + (info "no layout changes to apply")))) + +(defn reconfigure-subset + "Reconfigure cluster with only a subset of nodes" + [cnt] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] op + (case (:f op) + :start + (let [[keep-nodes remove-nodes] + (->> (:nodes test) + shuffle + (split-at cnt))] + (info "layout split: keep " keep-nodes ", remove " remove-nodes) + (configure-present! test keep-nodes) + (configure-absent! test remove-nodes) + (finalize-config! test) + (assoc op :value keep-nodes)) + :stop + (do + (info "layout un-split: all nodes=" (:nodes test)) + (configure-present! test (:nodes test)) + (finalize-config! test) + (assoc op :value (:nodes test))))) + + (teardown! [this test] this))) + +; ---- nemesis scenari ---- + +(defn nemesis-op + "A generator for a single nemesis operation" + [op] + (fn [_ _] {:type :info, :f op})) + +(defn reconfiguration-package + "Cluster reconfiguration nemesis package" + [opts] + {:generator (->> + (gen/mix [(nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger (:interval opts 5))) + :final-generator {:type :info, :f :reconfigure-stop} + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)}) + :perf #{{:name "reconfigure" + :start #{:reconfigure-start} + :stop #{:reconfigur-stop} + :color "#A197E9"}}}) + +(defn scenario-c + "Clock modifying scenario" + [opts] + (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})) + +(defn scenario-cp + "Clock modifying + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-r + "Cluster reconfiguration scenario" + [opts] + (reconfiguration-package {:interval 1})) + +(defn scenario-pr + "Partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cpr + "Clock scramble + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cdp + "Clock modifying + db + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-dpr + "Db + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj new file mode 100644 index 00000000..39708c0b --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -0,0 +1,143 @@ +(ns jepsen.garage.reg + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-get [_ _] {:type :invoke, :f :read, :value nil}) +(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))}) +(defn op-del [_ _] {:type :invoke, :f :write, :value nil}) + +(defrecord RegClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op)] + (case (:f op) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (let [value (s3/get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value)))) + :write + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) k v) + (assoc op :type :ok))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn reg-read-after-write + "Read-after-Write checker for register operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:put-values {-1 nil} + :put-done #{-1} + :put-in-progress {} + :read-can-contain {} + :bad-reads #{}} + final (reduce + (fn [state op] + (let [current-values (set/union + (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state))) + (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state)))) + read-can-contain (reduce + (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v))) + {} (:read-can-contain state))] + (info "--------") + (info "state: " state) + (info "current-values: " current-values) + (info "read-can-contain: " read-can-contain) + (info "op: " op) + (case [(:type op) (:f op)] + ([:invoke :write]) + (assoc state + :read-can-contain read-can-contain + :put-values (assoc (:put-values state) (:index op) (:value op)) + :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)])) + ([:ok :write]) + (let [[index overwrites] (get (:put-in-progress state) (:process op))] + (assoc state + :read-can-contain read-can-contain + :put-in-progress (dissoc (:put-in-progress state) (:process op)) + :put-done + (conj + (set/difference (:put-done state) overwrites) + index))) + ([:invoke :read]) + (assoc state + :read-can-contain (assoc read-can-contain (:process op) current-values)) + ([:ok :read]) + (let [this-read-can-contain (get read-can-contain (:process op)) + bad-reads (if (contains? this-read-can-contain (:value op)) + (:bad-reads state) + (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))] + (info "this-read-can-contain: " this-read-can-contain) + (assoc state + :read-can-contain (dissoc read-can-contain (:process op)) + :bad-reads bad-reads)) + state))) + init history) + valid? (empty? (:bad-reads final))] + (assoc final :valid? valid?))))) + +(defn workload-common + "Common parts of workload" + [opts] + {:client (RegClient. nil) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> + (gen/mix [op-get op-put op-del]) + (gen/limit (:ops-per-key opts)))))}) + +(defn workload1 + "Tests linearizable reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable + {:model (model/register) + :algorithm :linear}) + :timeline (timeline/html)})))) + +(defn workload2 + "Tests CRDT reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:reg-read-after-write (reg-read-after-write) + :timeline (timeline/html)})))) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj new file mode 100644 index 00000000..531e0157 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -0,0 +1,48 @@ +(ns jepsen.garage.s3api + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c]] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; GARAGE S3 HELPER FUNCTIONS + +(defn get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn list-inner [creds prefix ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner creds prefix (:next-continuation-token list-result) objects) + objects))) +(defn list + "Helper for ListObjects -- just lists everything in the bucket" + [creds prefix] + (list-inner creds prefix nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj new file mode 100644 index 00000000..2c7a2ccd --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -0,0 +1,135 @@ +(ns jepsen.garage.set + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [checker :as checker] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)}) +(defn op-read [_ _] {:type :invoke, :f :read, :value nil}) + +(defrecord SetClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op) + prefix (str "set" k "/")] + (case (:f op) + :add + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) (str prefix v) "present") + (assoc op :type :ok))) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (do + (let [items (s3/list (:creds this) prefix)] + (let [items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn set-read-after-write + "Read-after-Write checker for set operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:add-started #{} + :add-done #{} + :read-must-contain {} + :missed #{} + :unexpected #{}} + final (reduce + (fn [state op] + (case [(:type op) (:f op)] + ([:invoke :add]) + (assoc state :add-started (conj (:add-started state) (:value op))) + ([:ok :add]) + (assoc state :add-done (conj (:add-done state) (:value op))) + ([:invoke :read]) + (assoc-in state [:read-must-contain (:process op)] (:add-done state)) + ([:ok :read]) + (let [read-must-contain (get (:read-must-contain state) (:process op)) + new-missed (set/difference read-must-contain (:value op)) + new-unexpected (set/difference (:value op) (:add-started state))] + (assoc state + :read-must-contain (dissoc (:read-must-contain state) (:process op)) + :missed (set/union (:missed state) new-missed), + :unexpected (set/union (:unexpected state) new-unexpected))) + state)) + init history) + valid? (and (empty? (:missed final)) (empty? (:unexpected final)))] + (assoc final :valid? valid?))))) + +(defn workload1 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set (checker/set) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x})) + (gen/limit (:ops-per-key opts))))) + :final-generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (gen/phases + (gen/once op-read) + (gen/sleep 5))))}) + +(defn workload2 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set-read-after-write (set-read-after-write) + ; :set-full (checker/set-full {:linearizable? false}) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> (gen/mix [op-add-rand100 op-read]) + (gen/limit (:ops-per-key opts)))))}) + + diff --git a/script/jepsen.garage/test/jepsen/garage_test.clj b/script/jepsen.garage/test/jepsen/garage_test.clj new file mode 100644 index 00000000..055392a1 --- /dev/null +++ b/script/jepsen.garage/test/jepsen/garage_test.clj @@ -0,0 +1,7 @@ +(ns jepsen.garage-test + (:require [clojure.test :refer :all] + [jepsen.garage :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1)))) diff --git a/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml b/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml new file mode 100644 index 00000000..cd0fb166 --- /dev/null +++ b/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: garagenodes.deuxfleurs.fr +spec: + conversion: + strategy: None + group: deuxfleurs.fr + names: + kind: GarageNode + listKind: GarageNodeList + plural: garagenodes + singular: garagenode + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for Node via `CustomResource` + properties: + spec: + properties: + address: + format: ip + type: string + hostname: + type: string + port: + format: uint16 + minimum: 0 + type: integer + required: + - address + - hostname + - port + type: object + required: + - spec + title: GarageNode + type: object + served: true + storage: true + subresources: {} \ No newline at end of file diff --git a/script/k8s/crd/kustomization.yaml b/script/k8s/crd/kustomization.yaml new file mode 100644 index 00000000..9f20eccf --- /dev/null +++ b/script/k8s/crd/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- garagenodes.deuxfleurs.fr.yaml \ No newline at end of file diff --git a/script/not-dynamic.sh b/script/not-dynamic.sh index b9a13070..2beb4e0b 100755 --- a/script/not-dynamic.sh +++ b/script/not-dynamic.sh @@ -7,7 +7,12 @@ if [ "$#" -ne 1 ]; then exit 2 fi -if file $1 | grep 'dynamically linked' 2>&1; then +if [ ! -x "$1" ]; then + echo "[fail] $1 does not exist or is not an executable" + exit 1 +fi + +if file "$1" | grep 'dynamically linked' 2>&1; then echo "[fail] $1 is dynamic" exit 1 fi diff --git a/script/test-smoke.sh b/script/test-smoke.sh index 6965c0f3..acf56a90 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -81,10 +81,21 @@ if [ -z "$SKIP_AWS" ]; then echo "Invalid multipart upload" exit 1 fi -fi + aws s3api delete-object --bucket eprouvette --key upload -echo "OK!!" -exit 0 + echo "🛠️ Test SSE-C with awscli (aws s3)" + SSEC_KEY="u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y=" + SSEC_KEY_MD5="jMGbs3GyZkYjJUP6q5jA7g==" + echo "$SSEC_KEY" | base64 -d > /tmp/garage.ssec-key + for idx in {1,2}.rnd; do + aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \ + "/tmp/garage.$idx" "s3://eprouvette/garage.$idx.aws.sse-c" + aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \ + "s3://eprouvette/garage.$idx.aws.sse-c" "/tmp/garage.$idx.dl.sse-c" + diff "/tmp/garage.$idx" "/tmp/garage.$idx.dl.sse-c" + aws s3api delete-object --bucket eprouvette --key "garage.$idx.aws.sse-c" + done +fi # S3CMD if [ -z "$SKIP_S3CMD" ]; then diff --git a/script/test-upgrade.sh b/script/test-upgrade.sh new file mode 100755 index 00000000..dc25e7c6 --- /dev/null +++ b/script/test-upgrade.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -ex + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +SCRIPT_FOLDER="`dirname \"$0\"`" +REPO_FOLDER="${SCRIPT_FOLDER}/../" +GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" +GARAGE_RELEASE="${REPO_FOLDER}/target/release/" +NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/" +PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" + +OLD_VERSION="$1" +ARCH="$2" + + +echo "Downloading old garage binary..." +curl https://garagehq.deuxfleurs.fr/_releases/$OLD_VERSION/$ARCH/garage > /tmp/old_garage +chmod +x /tmp/old_garage + +echo "============= insert data into old version cluster =================" + +export GARAGE_BIN=/tmp/old_garage +if echo $OLD_VERSION | grep 'v0\.8\.'; then + echo "Detected Garage v0.8.x" + export GARAGE_08=1 +fi + +echo "⏳ Setup cluster using old version" +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-clean.sh +${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 & +sleep 6 +${SCRIPT_FOLDER}/dev-configure.sh +${SCRIPT_FOLDER}/dev-bucket.sh + +echo "🛠️ Inserting data in old cluster" +source ${SCRIPT_FOLDER}/dev-env-rclone.sh +rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line + +echo "🏁 Stopping old cluster" +killall -INT old_garage +sleep 2 +killall -9 old_garage || true + +echo "🏁 Removing old garage version" +rm -rv $GARAGE_BIN +export -n GARAGE_BIN +export -n GARAGE_08 + +echo "================ read data from new cluster ===================" + +echo "⏳ Setup cluster using new version" +pwd +ls +export GARAGE_BIN=$(which garage) +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 & +sleep 3 + +echo "🛠️ Retrieving data from old cluster" +rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list + +if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then + echo "TEST FAILURE: directories are different" + exit 1 +fi +rm -r /tmp/test_dotgit + +echo "🏁 Teardown" +rm -rf /tmp/garage-{data,meta}-* +rm -rf /tmp/config.*.toml + +echo "✅ Success" diff --git a/shell.nix b/shell.nix index 0d510e33..c3dedca8 100644 --- a/shell.nix +++ b/shell.nix @@ -3,101 +3,53 @@ with import ./nix/common.nix; let - pkgs = import pkgsSrc { + pkgs = import nixpkgs { inherit system; - overlays = [ cargo2nixOverlay ]; }; - kaniko = (import ./nix/kaniko.nix) pkgs; - manifest-tool = (import ./nix/manifest-tool.nix) pkgs; winscp = (import ./nix/winscp.nix) pkgs; +in +{ + # --- Dev shell inherited from flake.nix --- + devShell = devShells.default; + devShellFull = devShells.full; -in { - # --- Rust Shell --- - # Use it to compile Garage - rust = pkgs.mkShell { - nativeBuildInputs = [ - #pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - #pkgs.clippy - pkgs.rustfmt - #pkgs.perl - #pkgs.protobuf - #pkgs.pkg-config - #pkgs.openssl - pkgs.file - #cargo2nix.packages.x86_64-linux.cargo2nix - ]; - }; - - # --- Integration shell --- - # Use it to test Garage with common S3 clients - integration = pkgs.mkShell { - nativeBuildInputs = [ + # --- Continuous integration shell --- + # The shell used for all CI jobs (along with devShell) + ci = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ winscp - pkgs.s3cmd - pkgs.awscli2 - pkgs.minio-client - pkgs.rclone - pkgs.socat - pkgs.psmisc - pkgs.which - pkgs.openssl - pkgs.curl - pkgs.jq + + kaniko + manifest-tool + awscli2 + file + s3cmd + minio-client + rclone + socat + psmisc + which + openssl + curl + jq ]; - }; - - # --- Release shell --- - # A shell built to make releasing easier - release = pkgs.mkShell { shellHook = '' - function refresh_toolchain { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec - } - - function refresh_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do - echo "Updating cache for ''${attr}" - derivation=$(nix-instantiate --attr ''${attr}) - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation%\!bin}) - done - rm /tmp/nix-signing-key.sec - } - - function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default devShell.x86_64-linux; do - echo "Updating cache for ''${attr}" - derivation=$(nix path-info --derivation ".#''${attr}") - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation}) - done - rm /tmp/nix-signing-key.sec - } + export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' function to_s3 { aws \ --endpoint-url https://garage.deuxfleurs.fr \ --region garage \ s3 cp \ - ./result-bin/bin/garage \ - s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage + ./result/bin/garage \ + s3://garagehq.deuxfleurs.fr/_releases/''${CI_COMMIT_TAG:-$CI_COMMIT_SHA}/''${TARGET}/garage } function to_docker { executor \ --force \ - --customPlatform="''${DOCKER_PLATFORM}" \ - --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ + --customPlatform="$(echo "''${DOCKER_PLATFORM}" | sed 's/i386/386/')" \ + --destination "$(echo "''${CONTAINER_NAME}" | sed 's/i386/386/'):''${CONTAINER_TAG}" \ --context dir://`pwd` \ --verbosity=debug } @@ -156,7 +108,25 @@ in { s3://garagehq.deuxfleurs.fr/ } ''; - nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ]; + + }; + + # --- Cache shell --- + # A shell for refreshing caches + cache = pkgs.mkShell { + shellHook = '' + function refresh_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in pkgs.amd64.debug test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do + echo "Updating cache for ''${attr}" + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') + + done + rm /tmp/nix-signing-key.sec + } + ''; }; } diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml deleted file mode 100644 index 6425591f..00000000 --- a/src/api/Cargo.toml +++ /dev/null @@ -1,63 +0,0 @@ -[package] -name = "garage_api" -version = "0.8.4" -authors = ["Alex Auvolat "] -edition = "2018" -license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" -repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" -readme = "../../README.md" - -[lib] -path = "lib.rs" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -garage_model.workspace = true -garage_table.workspace = true -garage_block.workspace = true -garage_util.workspace = true -garage_rpc.workspace = true - -async-trait = "0.1.7" -base64 = "0.21" -bytes = "1.0" -chrono = "0.4" -crypto-common = "0.1" -err-derive = "0.3" -hex = "0.4" -hmac = "0.12" -idna = "0.4" -tracing = "0.1" -md-5 = "0.10" -nom = "7.1" -sha2 = "0.10" - -futures = "0.3" -futures-util = "0.3" -pin-project = "1.0.12" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-stream = "0.1" - -form_urlencoded = "1.0.0" -http = "0.2" -httpdate = "1.0" -http-range = "0.1" -hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } -multer = "2.0" -percent-encoding = "2.1.0" -roxmltree = "0.18" -serde = { version = "1.0", features = ["derive"] } -serde_bytes = "0.11" -serde_json = "1.0" -quick-xml = { version = "0.26", features = [ "serialize" ] } -url = "2.3" - -opentelemetry = "0.17" -opentelemetry-prometheus = { version = "0.10", optional = true } -prometheus = { version = "0.13", optional = true } - -[features] -k2v = [ "garage_util/k2v", "garage_model/k2v" ] -metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml new file mode 100644 index 00000000..656c6825 --- /dev/null +++ b/src/api/admin/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "garage_api_admin" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Admin API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +argon2.workspace = true +async-trait.workspace = true +thiserror.workspace = true +hex.workspace = true +tracing.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } + +[features] +metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index cc04d81f..6f0c474f 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -1,12 +1,11 @@ use std::collections::HashMap; -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; +use argon2::password_hash::PasswordHash; -use futures::future::Future; use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW}; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; +use tokio::sync::watch; use opentelemetry::trace::SpanRef; @@ -18,15 +17,19 @@ use prometheus::{Encoder, TextEncoder}; use garage_model::garage::Garage; use garage_rpc::system::ClusterHealthStatus; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; -use crate::generic_server::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; -use crate::admin::bucket::*; -use crate::admin::cluster::*; -use crate::admin::error::*; -use crate::admin::key::*; -use crate::admin::router::{Authorization, Endpoint}; -use crate::helpers::host_to_bucket; +use crate::bucket::*; +use crate::cluster::*; +use crate::error::*; +use crate::key::*; +use crate::router_v0; +use crate::router_v1::{Authorization, Endpoint}; + +pub type ResBody = BoxBody; pub struct AdminApiServer { garage: Arc, @@ -42,14 +45,8 @@ impl AdminApiServer { #[cfg(feature = "metrics")] exporter: PrometheusExporter, ) -> Self { let cfg = &garage.config.admin; - let metrics_token = cfg - .metrics_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); - let admin_token = cfg - .admin_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); + let metrics_token = cfg.metrics_token.as_deref().map(hash_bearer_token); + let admin_token = cfg.admin_token.as_deref().map(hash_bearer_token); Self { garage, #[cfg(feature = "metrics")] @@ -61,25 +58,28 @@ impl AdminApiServer { pub async fn run( self, - bind_addr: SocketAddr, - shutdown_signal: impl Future, + bind_addr: UnixOrTCPSocketAddress, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { let region = self.garage.config.s3_api.s3_region.clone(); ApiServer::new(region, self) - .run_server(bind_addr, shutdown_signal) + .run_server(bind_addr, Some(0o220), must_exit) .await } - fn handle_options(&self, _req: &Request) -> Result, Error> { + fn handle_options(&self, _req: &Request) -> Result, Error> { Ok(Response::builder() .status(StatusCode::NO_CONTENT) .header(ALLOW, "OPTIONS, GET, POST") .header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS, GET, POST") .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .body(Body::empty())?) + .body(empty_body())?) } - async fn handle_check_domain(&self, req: Request) -> Result, Error> { + async fn handle_check_domain( + &self, + req: Request, + ) -> Result, Error> { let query_params: HashMap = req .uri() .query() @@ -103,7 +103,7 @@ impl AdminApiServer { if self.check_domain(domain).await? { Ok(Response::builder() .status(StatusCode::OK) - .body(Body::from(format!( + .body(string_body(format!( "Domain '{domain}' is managed by Garage" )))?) } else { @@ -166,7 +166,7 @@ impl AdminApiServer { } } - fn handle_health(&self) -> Result, Error> { + fn handle_health(&self) -> Result, Error> { let health = self.garage.system.health(); let (status, status_str) = match health.status { @@ -181,17 +181,17 @@ impl AdminApiServer { ), }; let status_str = format!( - "{}\nConsult the full health check API endpoint at /v0/health for more details\n", + "{}\nConsult the full health check API endpoint at /v1/health for more details\n", status_str ); Ok(Response::builder() .status(status) .header(http::header::CONTENT_TYPE, "text/plain") - .body(Body::from(status_str))?) + .body(string_body(status_str))?) } - fn handle_metrics(&self) -> Result, Error> { + fn handle_metrics(&self) -> Result, Error> { #[cfg(feature = "metrics")] { use opentelemetry::trace::Tracer; @@ -211,7 +211,7 @@ impl AdminApiServer { Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, encoder.format_type()) - .body(Body::from(buffer))?) + .body(bytes_body(buffer.into()))?) } #[cfg(not(feature = "metrics"))] Err(Error::bad_request( @@ -220,7 +220,6 @@ impl AdminApiServer { } } -#[async_trait] impl ApiHandler for AdminApiServer { const API_NAME: &'static str = "admin"; const API_NAME_DISPLAY: &'static str = "Admin"; @@ -228,20 +227,25 @@ impl ApiHandler for AdminApiServer { type Endpoint = Endpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { - Endpoint::from_request(req) + fn parse_endpoint(&self, req: &Request) -> Result { + if req.uri().path().starts_with("/v0/") { + let endpoint_v0 = router_v0::Endpoint::from_request(req)?; + Endpoint::from_v0(endpoint_v0) + } else { + Endpoint::from_request(req) + } } async fn handle( &self, - req: Request, + req: Request, endpoint: Endpoint, - ) -> Result, Error> { - let expected_auth_header = + ) -> Result, Error> { + let required_auth_hash = match endpoint.authorization_type() { Authorization::None => None, - Authorization::MetricsToken => self.metrics_token.as_ref(), - Authorization::AdminToken => match &self.admin_token { + Authorization::MetricsToken => self.metrics_token.as_deref(), + Authorization::AdminToken => match self.admin_token.as_deref() { None => return Err(Error::forbidden( "Admin token isn't configured, admin API access is disabled for security.", )), @@ -249,14 +253,11 @@ impl ApiHandler for AdminApiServer { }, }; - if let Some(h) = expected_auth_header { + if let Some(password_hash) = required_auth_hash { match req.headers().get("Authorization") { None => return Err(Error::forbidden("Authorization token must be provided")), - Some(v) => { - let authorized = v.to_str().map(|hv| hv.trim() == h).unwrap_or(false); - if !authorized { - return Err(Error::forbidden("Invalid authorization token provided")); - } + Some(authorization) => { + verify_bearer_token(&authorization, password_hash)?; } } } @@ -273,7 +274,7 @@ impl ApiHandler for AdminApiServer { Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await, - Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await, + Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage).await, // Keys Endpoint::ListKeys => handle_list_keys(&self.garage).await, Endpoint::GetKeyInfo { @@ -331,3 +332,35 @@ impl ApiEndpoint for Endpoint { fn add_span_attributes(&self, _span: SpanRef<'_>) {} } + +fn hash_bearer_token(token: &str) -> String { + use argon2::{ + password_hash::{rand_core::OsRng, PasswordHasher, SaltString}, + Argon2, + }; + + let salt = SaltString::generate(&mut OsRng); + let argon2 = Argon2::default(); + argon2 + .hash_password(token.trim().as_bytes(), &salt) + .expect("could not hash API token") + .to_string() +} + +fn verify_bearer_token(token: &hyper::http::HeaderValue, password_hash: &str) -> Result<(), Error> { + use argon2::{password_hash::PasswordVerifier, Argon2}; + + let parsed_hash = PasswordHash::new(&password_hash).unwrap(); + + token + .to_str()? + .strip_prefix("Bearer ") + .and_then(|token| { + Argon2::default() + .verify_password(token.trim().as_bytes(), &parsed_hash) + .ok() + }) + .ok_or_else(|| Error::forbidden("Invalid authorization token"))?; + + Ok(()) +} diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 17f46c30..207693b6 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use garage_util::crdt::*; @@ -17,12 +17,14 @@ use garage_model::permission::*; use garage_model::s3::mpu_table; use garage_model::s3::object_table::*; -use crate::admin::error::*; -use crate::admin::key::ApiBucketKeyPerm; -use crate::common_error::CommonError; -use crate::helpers::{json_ok_response, parse_json_body}; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; -pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { +use crate::api_server::ResBody; +use crate::error::*; +use crate::key::ApiBucketKeyPerm; + +pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage .bucket_table .get_range( @@ -90,7 +92,7 @@ pub async fn handle_get_bucket_info( garage: &Arc, id: Option, global_alias: Option, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = match (id, global_alias) { (Some(id), None) => parse_bucket_id(&id)?, (None, Some(ga)) => garage @@ -111,7 +113,7 @@ pub async fn handle_get_bucket_info( async fn bucket_info_results( garage: &Arc, bucket_id: Uuid, -) -> Result, Error> { +) -> Result, Error> { let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) @@ -122,7 +124,7 @@ async fn bucket_info_results( .table .get(&bucket_id, &EmptyKey) .await? - .map(|x| x.filtered_values(&garage.system.ring.borrow())) + .map(|x| x.filtered_values(&garage.system.cluster_layout())) .unwrap_or_default(); let mpu_counters = garage @@ -130,7 +132,7 @@ async fn bucket_info_results( .table .get(&bucket_id, &EmptyKey) .await? - .map(|x| x.filtered_values(&garage.system.ring.borrow())) + .map(|x| x.filtered_values(&garage.system.cluster_layout())) .unwrap_or_default(); let mut relevant_keys = HashMap::new(); @@ -268,12 +270,14 @@ struct GetBucketInfoKey { pub async fn handle_create_bucket( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let helper = garage.locked_helper().await; if let Some(ga) = &req.global_alias { - if !is_valid_bucket_name(ga) { + if !is_valid_bucket_name(ga, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", ga, INVALID_BUCKET_NAME_MESSAGE @@ -288,17 +292,14 @@ pub async fn handle_create_bucket( } if let Some(la) = &req.local_alias { - if !is_valid_bucket_name(&la.alias) { + if !is_valid_bucket_name(&la.alias, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", la.alias, INVALID_BUCKET_NAME_MESSAGE ))); } - let key = garage - .key_helper() - .get_existing_key(&la.access_key_id) - .await?; + let key = helper.key().get_existing_key(&la.access_key_id).await?; let state = key.state.as_option().unwrap(); if matches!(state.local_aliases.get(&la.alias), Some(_)) { return Err(Error::bad_request("Local alias already exists")); @@ -309,21 +310,16 @@ pub async fn handle_create_bucket( garage.bucket_table.insert(&bucket).await?; if let Some(ga) = &req.global_alias { - garage - .bucket_helper() - .set_global_bucket_alias(bucket.id, ga) - .await?; + helper.set_global_bucket_alias(bucket.id, ga).await?; } if let Some(la) = &req.local_alias { - garage - .bucket_helper() + helper .set_local_bucket_alias(bucket.id, &la.access_key_id, &la.alias) .await?; if la.allow.read || la.allow.write || la.allow.owner { - garage - .bucket_helper() + helper .set_bucket_key_permissions( bucket.id, &la.access_key_id, @@ -360,16 +356,16 @@ struct CreateBucketLocalAlias { pub async fn handle_delete_bucket( garage: &Arc, id: String, -) -> Result, Error> { - let helper = garage.bucket_helper(); +) -> Result, Error> { + let helper = garage.locked_helper().await; let bucket_id = parse_bucket_id(&id)?; - let mut bucket = helper.get_existing_bucket(bucket_id).await?; + let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?; let state = bucket.state.as_option().unwrap(); // Check bucket is empty - if !helper.is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } @@ -386,7 +382,7 @@ pub async fn handle_delete_bucket( for ((key_id, alias), _, active) in state.local_aliases.items().iter() { if *active { helper - .unset_local_bucket_alias(bucket.id, key_id, alias) + .purge_local_bucket_alias(bucket.id, key_id, alias) .await?; } } @@ -403,15 +399,15 @@ pub async fn handle_delete_bucket( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_update_bucket( garage: &Arc, id: String, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let bucket_id = parse_bucket_id(&id)?; let mut bucket = garage @@ -470,23 +466,19 @@ struct UpdateBucketWebsiteAccess { pub async fn handle_bucket_change_key_perm( garage: &Arc, - req: Request, + req: Request, new_perm_flag: bool, -) -> Result, Error> { - let req = parse_json_body::(req).await?; +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let helper = garage.locked_helper().await; let bucket_id = parse_bucket_id(&req.bucket_id)?; - let bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; + let bucket = helper.bucket().get_existing_bucket(bucket_id).await?; let state = bucket.state.as_option().unwrap(); - let key = garage - .key_helper() - .get_existing_key(&req.access_key_id) - .await?; + let key = helper.key().get_existing_key(&req.access_key_id).await?; let mut perm = state .authorized_keys @@ -504,8 +496,7 @@ pub async fn handle_bucket_change_key_perm( perm.allow_owner = new_perm_flag; } - garage - .bucket_helper() + helper .set_bucket_key_permissions(bucket.id, &key.key_id, perm) .await?; @@ -526,13 +517,12 @@ pub async fn handle_global_alias_bucket( garage: &Arc, bucket_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() - .set_global_bucket_alias(bucket_id, &alias) - .await?; + let helper = garage.locked_helper().await; + + helper.set_global_bucket_alias(bucket_id, &alias).await?; bucket_info_results(garage, bucket_id).await } @@ -541,13 +531,12 @@ pub async fn handle_global_unalias_bucket( garage: &Arc, bucket_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &alias) - .await?; + let helper = garage.locked_helper().await; + + helper.unset_global_bucket_alias(bucket_id, &alias).await?; bucket_info_results(garage, bucket_id).await } @@ -557,11 +546,12 @@ pub async fn handle_local_alias_bucket( bucket_id: String, access_key_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() + let helper = garage.locked_helper().await; + + helper .set_local_bucket_alias(bucket_id, &access_key_id, &alias) .await?; @@ -573,11 +563,12 @@ pub async fn handle_local_unalias_bucket( bucket_id: String, access_key_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() + let helper = garage.locked_helper().await; + + helper .unset_local_bucket_alias(bucket_id, &access_key_id, &alias) .await?; diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index c8107b82..ffa0fa71 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -1,7 +1,8 @@ +use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use hyper::{Body, Request, Response}; +use hyper::{body::Incoming as IncomingBody, Request, Response}; use serde::{Deserialize, Serialize}; use garage_util::crdt::*; @@ -11,35 +12,111 @@ use garage_rpc::layout; use garage_model::garage::Garage; -use crate::admin::error::*; -use crate::helpers::{json_ok_response, parse_json_body}; +use garage_api_common::helpers::{json_ok_response, parse_json_body}; + +use crate::api_server::ResBody; +use crate::error::*; + +pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { + let layout = garage.system.cluster_layout(); + let mut nodes = garage + .system + .get_known_nodes() + .into_iter() + .map(|i| { + ( + i.id, + NodeResp { + id: hex::encode(i.id), + addr: i.addr, + hostname: i.status.hostname, + is_up: i.is_up, + last_seen_secs_ago: i.last_seen_secs_ago, + data_partition: i + .status + .data_disk_avail + .map(|(avail, total)| FreeSpaceResp { + available: avail, + total, + }), + metadata_partition: i.status.meta_disk_avail.map(|(avail, total)| { + FreeSpaceResp { + available: avail, + total, + } + }), + ..Default::default() + }, + ) + }) + .collect::>(); + + for (id, _, role) in layout.current().roles.items().iter() { + if let layout::NodeRoleV(Some(r)) = role { + let role = NodeRoleResp { + id: hex::encode(id), + zone: r.zone.to_string(), + capacity: r.capacity, + tags: r.tags.clone(), + }; + match nodes.get_mut(id) { + None => { + nodes.insert( + *id, + NodeResp { + id: hex::encode(id), + role: Some(role), + ..Default::default() + }, + ); + } + Some(n) => { + n.role = Some(role); + } + } + } + } + + for ver in layout.versions().iter().rev().skip(1) { + for (id, _, role) in ver.roles.items().iter() { + if let layout::NodeRoleV(Some(r)) = role { + if r.capacity.is_some() { + if let Some(n) = nodes.get_mut(id) { + if n.role.is_none() { + n.draining = true; + } + } else { + nodes.insert( + *id, + NodeResp { + id: hex::encode(id), + draining: true, + ..Default::default() + }, + ); + } + } + } + } + } + + let mut nodes = nodes.into_values().collect::>(); + nodes.sort_by(|x, y| x.id.cmp(&y.id)); -pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { node: hex::encode(garage.system.id), garage_version: garage_util::version::garage_version(), garage_features: garage_util::version::garage_features(), rust_version: garage_util::version::rust_version(), db_engine: garage.db.engine(), - known_nodes: garage - .system - .get_known_nodes() - .into_iter() - .map(|i| KnownNodeResp { - id: hex::encode(i.id), - addr: i.addr, - is_up: i.is_up, - last_seen_secs_ago: i.last_seen_secs_ago, - hostname: i.status.hostname, - }) - .collect(), - layout: format_cluster_layout(&garage.system.get_cluster_layout()), + layout_version: layout.current().version, + nodes, }; Ok(json_ok_response(&res)?) } -pub async fn handle_get_cluster_health(garage: &Arc) -> Result, Error> { +pub async fn handle_get_cluster_health(garage: &Arc) -> Result, Error> { use garage_rpc::system::ClusterHealthStatus; let health = garage.system.health(); let health = ClusterHealth { @@ -61,9 +138,9 @@ pub async fn handle_get_cluster_health(garage: &Arc) -> Result, - req: Request, -) -> Result, Error> { - let req = parse_json_body::>(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::, _, Error>(req).await?; let res = futures::future::join_all(req.iter().map(|node| garage.system.connect(node))) .await @@ -83,14 +160,15 @@ pub async fn handle_connect_cluster_nodes( Ok(json_ok_response(&res)?) } -pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { - let res = format_cluster_layout(&garage.system.get_cluster_layout()); +pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { + let res = format_cluster_layout(garage.system.cluster_layout().inner()); Ok(json_ok_response(&res)?) } -fn format_cluster_layout(layout: &layout::ClusterLayout) -> GetClusterLayoutResponse { +fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResponse { let roles = layout + .current() .roles .items() .iter() @@ -104,10 +182,12 @@ fn format_cluster_layout(layout: &layout::ClusterLayout) -> GetClusterLayoutResp .collect::>(); let staged_role_changes = layout - .staging_roles + .staging + .get() + .roles .items() .iter() - .filter(|(k, _, v)| layout.roles.get(k) != Some(v)) + .filter(|(k, _, v)| layout.current().roles.get(k) != Some(v)) .map(|(k, _, v)| match &v.0 { None => NodeRoleChange { id: hex::encode(k), @@ -125,7 +205,7 @@ fn format_cluster_layout(layout: &layout::ClusterLayout) -> GetClusterLayoutResp .collect::>(); GetClusterLayoutResponse { - version: layout.version, + version: layout.current().version, roles, staged_role_changes, } @@ -154,8 +234,8 @@ struct GetClusterStatusResponse { garage_features: Option<&'static [&'static str]>, rust_version: &'static str, db_engine: String, - known_nodes: Vec, - layout: GetClusterLayoutResponse, + layout_version: u64, + nodes: Vec, } #[derive(Serialize)] @@ -189,28 +269,41 @@ struct NodeRoleResp { tags: Vec, } -#[derive(Serialize)] +#[derive(Serialize, Default)] #[serde(rename_all = "camelCase")] -struct KnownNodeResp { +struct FreeSpaceResp { + available: u64, + total: u64, +} + +#[derive(Serialize, Default)] +#[serde(rename_all = "camelCase")] +struct NodeResp { id: String, - addr: SocketAddr, + role: Option, + addr: Option, + hostname: Option, is_up: bool, last_seen_secs_ago: Option, - hostname: String, + draining: bool, + #[serde(skip_serializing_if = "Option::is_none")] + data_partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + metadata_partition: Option, } // ---- update functions ---- pub async fn handle_update_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let updates = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let updates = parse_json_body::(req).await?; - let mut layout = garage.system.get_cluster_layout(); + let mut layout = garage.system.cluster_layout().inner().clone(); - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging_roles); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); for change in updates { let node = hex::decode(&change.id).ok_or_bad_request("Invalid node identifier")?; @@ -231,11 +324,17 @@ pub async fn handle_update_cluster_layout( }; layout - .staging_roles + .staging + .get_mut() + .roles .merge(&roles.update_mutator(node, layout::NodeRoleV(new_role))); } - garage.system.update_cluster_layout(&layout).await?; + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; let res = format_cluster_layout(&layout); Ok(json_ok_response(&res)?) @@ -243,14 +342,18 @@ pub async fn handle_update_cluster_layout( pub async fn handle_apply_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let param = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let param = parse_json_body::(req).await?; - let layout = garage.system.get_cluster_layout(); + let layout = garage.system.cluster_layout().inner().clone(); let (layout, msg) = layout.apply_staged_changes(Some(param.version))?; - garage.system.update_cluster_layout(&layout).await?; + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; let res = ApplyClusterLayoutResponse { message: msg, @@ -261,13 +364,14 @@ pub async fn handle_apply_cluster_layout( pub async fn handle_revert_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let param = parse_json_body::(req).await?; - - let layout = garage.system.get_cluster_layout(); - let layout = layout.revert_staged_changes(Some(param.version))?; - garage.system.update_cluster_layout(&layout).await?; +) -> Result, Error> { + let layout = garage.system.cluster_layout().inner().clone(); + let layout = layout.revert_staged_changes()?; + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; let res = format_cluster_layout(&layout); Ok(json_ok_response(&res)?) @@ -279,7 +383,7 @@ type UpdateClusterLayoutRequest = Vec; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] -struct ApplyRevertLayoutRequest { +struct ApplyLayoutRequest { version: u64, } diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index ed1a07bd..17d4c200 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -1,53 +1,46 @@ -use err_derive::Error; +use std::convert::TryFrom; + use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; pub use garage_model::helper::error::Error as HelperError; -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::CustomApiErrorBody; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// The API access key does not exist - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), /// In Import key, the key already exists - #[error( - display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", - _0 - )] + #[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")] KeyAlreadyExists(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} +commonErrorDerivative!(Error); +/// FIXME: helper errors are transformed into their corresponding variants +/// in the Error struct, but in many case a helper error should be considered +/// an internal error. impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n), + fn from(err: HelperError) -> Error { + match CommonError::try_from(err) { + Ok(ce) => Self::Common(ce), + Err(HelperError::NoSuchAccessKey(k)) => Self::NoSuchAccessKey(k), + Err(_) => unreachable!(), } } } @@ -77,14 +70,14 @@ impl ApiError for Error { header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap()); } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = CustomApiErrorBody { code: self.code().to_string(), message: format!("{}", self), path: path.to_string(), region: garage_region.to_string(), }; - Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + let error_str = serde_json::to_string_pretty(&error).unwrap_or_else(|_| { r#" { "code": "InternalError", @@ -92,6 +85,7 @@ impl ApiError for Error { } "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 8d1c6890..bebf3063 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use garage_table::*; @@ -9,10 +9,12 @@ use garage_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; -use crate::admin::error::*; -use crate::helpers::{is_default, json_ok_response, parse_json_body}; +use garage_api_common::helpers::*; -pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { +use crate::api_server::ResBody; +use crate::error::*; + +pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage .key_table .get_range( @@ -45,7 +47,7 @@ pub async fn handle_get_key_info( id: Option, search: Option, show_secret_key: bool, -) -> Result, Error> { +) -> Result, Error> { let key = if let Some(id) = id { garage.key_helper().get_existing_key(&id).await? } else if let Some(search) = search { @@ -62,9 +64,9 @@ pub async fn handle_get_key_info( pub async fn handle_create_key( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let key = Key::new(req.name.as_deref().unwrap_or("Unnamed key")); garage.key_table.insert(&key).await?; @@ -80,9 +82,9 @@ struct CreateKeyRequest { pub async fn handle_import_key( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let prev_key = garage.key_table.get(&EmptyKey, &req.access_key_id).await?; if prev_key.is_some() { @@ -111,9 +113,9 @@ struct ImportKeyRequest { pub async fn handle_update_key( garage: &Arc, id: String, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let mut key = garage.key_helper().get_existing_key(&id).await?; @@ -146,23 +148,26 @@ struct UpdateKeyRequest { deny: Option, } -pub async fn handle_delete_key(garage: &Arc, id: String) -> Result, Error> { - let mut key = garage.key_helper().get_existing_key(&id).await?; +pub async fn handle_delete_key( + garage: &Arc, + id: String, +) -> Result, Error> { + let helper = garage.locked_helper().await; - key.state.as_option().unwrap(); + let mut key = helper.key().get_existing_key(&id).await?; - garage.key_helper().delete_key(&mut key).await?; + helper.delete_key(&mut key).await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } async fn key_info_results( garage: &Arc, key: Key, show_secret: bool, -) -> Result, Error> { +) -> Result, Error> { let mut relevant_buckets = HashMap::new(); let key_state = key.state.as_option().unwrap(); diff --git a/src/api/admin/mod.rs b/src/api/admin/lib.rs similarity index 50% rename from src/api/admin/mod.rs rename to src/api/admin/lib.rs index c4857c10..599e9b44 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/lib.rs @@ -1,6 +1,10 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; mod error; -mod router; +mod router_v0; +mod router_v1; mod bucket; mod cluster; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs deleted file mode 100644 index 254aff12..00000000 --- a/src/api/admin/router.rs +++ /dev/null @@ -1,160 +0,0 @@ -use std::borrow::Cow; - -use hyper::{Method, Request}; - -use crate::admin::error::*; -use crate::router_macros::*; - -pub enum Authorization { - None, - MetricsToken, - AdminToken, -} - -router_match! {@func - -/// List of all Admin API endpoints. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Endpoint { - Options, - CheckDomain, - Health, - Metrics, - GetClusterStatus, - GetClusterHealth, - ConnectClusterNodes, - // Layout - GetClusterLayout, - UpdateClusterLayout, - ApplyClusterLayout, - RevertClusterLayout, - // Keys - ListKeys, - CreateKey, - ImportKey, - GetKeyInfo { - id: Option, - search: Option, - show_secret_key: Option, - }, - DeleteKey { - id: String, - }, - UpdateKey { - id: String, - }, - // Buckets - ListBuckets, - CreateBucket, - GetBucketInfo { - id: Option, - global_alias: Option, - }, - DeleteBucket { - id: String, - }, - UpdateBucket { - id: String, - }, - // Bucket-Key Permissions - BucketAllowKey, - BucketDenyKey, - // Bucket aliases - GlobalAliasBucket { - id: String, - alias: String, - }, - GlobalUnaliasBucket { - id: String, - alias: String, - }, - LocalAliasBucket { - id: String, - access_key_id: String, - alias: String, - }, - LocalUnaliasBucket { - id: String, - access_key_id: String, - alias: String, - }, -}} - -impl Endpoint { - /// Determine which S3 endpoint a request is for using the request, and a bucket which was - /// possibly extracted from the Host header. - /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets - pub fn from_request(req: &Request) -> Result { - let uri = req.uri(); - let path = uri.path(); - let query = uri.query(); - - let mut query = QueryParameters::from_query(query.unwrap_or_default())?; - - let res = router_match!(@gen_path_parser (req.method(), path, query) [ - OPTIONS _ => Options, - GET "/check" => CheckDomain, - GET "/health" => Health, - GET "/metrics" => Metrics, - GET "/v1/status" => GetClusterStatus, - GET "/v1/health" => GetClusterHealth, - POST ("/v0/connect" | "/v1/connect") => ConnectClusterNodes, - // Layout endpoints - GET "/v1/layout" => GetClusterLayout, - POST "/v1/layout" => UpdateClusterLayout, - POST "/v1/layout/apply" => ApplyClusterLayout, - POST ("/v0/layout/revert" | "/v1/layout/revert") => RevertClusterLayout, - // API key endpoints - GET "/v1/key" if id => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), - GET "/v1/key" if search => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), - POST "/v1/key" if id => UpdateKey (query::id), - POST "/v1/key" => CreateKey, - POST "/v1/key/import" => ImportKey, - DELETE ("/v0/key" | "/v1/key") if id => DeleteKey (query::id), - GET ("/v0/key" | "/v1/key") => ListKeys, - // Bucket endpoints - GET ("/v0/bucket" | "/v1/bucket") if id => GetBucketInfo (query_opt::id, query_opt::global_alias), - GET ("/v0/bucket" | "/v1/bucket") if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), - GET ("/v0/bucket" | "/v1/bucket") => ListBuckets, - POST ("/v0/bucket" | "/v1/bucket") => CreateBucket, - DELETE ("/v0/bucket" | "/v1/bucket") if id => DeleteBucket (query::id), - PUT ("/v0/bucket" | "/v1/bucket") if id => UpdateBucket (query::id), - // Bucket-key permissions - POST ("/v0/bucket/allow" | "/v1/bucket/allow") => BucketAllowKey, - POST ("/v0/bucket/deny" | "/v1/bucket/deny") => BucketDenyKey, - // Bucket aliases - PUT ("/v0/bucket/alias/global" | "/v1/bucket/alias/global") => GlobalAliasBucket (query::id, query::alias), - DELETE ("/v0/bucket/alias/global" | "/v1/bucket/alias/global") => GlobalUnaliasBucket (query::id, query::alias), - PUT ("/v0/bucket/alias/local" | "/v1/bucket/alias/local") => LocalAliasBucket (query::id, query::access_key_id, query::alias), - DELETE ("/v0/bucket/alias/local" | "/v1/bucket/alias/local") => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), - ]); - - if let Some(message) = query.nonempty_message() { - debug!("Unused query parameter: {}", message) - } - - Ok(res) - } - /// Get the kind of authorization which is required to perform the operation. - pub fn authorization_type(&self) -> Authorization { - match self { - Self::Health => Authorization::None, - Self::CheckDomain => Authorization::None, - Self::Metrics => Authorization::MetricsToken, - _ => Authorization::AdminToken, - } - } -} - -generateQueryParameters! { - keywords: [], - fields: [ - "format" => format, - "id" => id, - "search" => search, - "globalAlias" => global_alias, - "alias" => alias, - "accessKeyId" => access_key_id, - "showSecretKey" => show_secret_key - ] -} diff --git a/src/api/admin/router_v0.rs b/src/api/admin/router_v0.rs new file mode 100644 index 00000000..9dd742ba --- /dev/null +++ b/src/api/admin/router_v0.rs @@ -0,0 +1,144 @@ +use std::borrow::Cow; + +use hyper::{Method, Request}; + +use garage_api_common::router_macros::*; + +use crate::error::*; + +router_match! {@func + +/// List of all Admin API endpoints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Endpoint { + Options, + CheckDomain, + Health, + Metrics, + GetClusterStatus, + GetClusterHealth, + ConnectClusterNodes, + // Layout + GetClusterLayout, + UpdateClusterLayout, + ApplyClusterLayout, + RevertClusterLayout, + // Keys + ListKeys, + CreateKey, + ImportKey, + GetKeyInfo { + id: Option, + search: Option, + }, + DeleteKey { + id: String, + }, + UpdateKey { + id: String, + }, + // Buckets + ListBuckets, + CreateBucket, + GetBucketInfo { + id: Option, + global_alias: Option, + }, + DeleteBucket { + id: String, + }, + UpdateBucket { + id: String, + }, + // Bucket-Key Permissions + BucketAllowKey, + BucketDenyKey, + // Bucket aliases + GlobalAliasBucket { + id: String, + alias: String, + }, + GlobalUnaliasBucket { + id: String, + alias: String, + }, + LocalAliasBucket { + id: String, + access_key_id: String, + alias: String, + }, + LocalUnaliasBucket { + id: String, + access_key_id: String, + alias: String, + }, +}} + +impl Endpoint { + /// Determine which S3 endpoint a request is for using the request, and a bucket which was + /// possibly extracted from the Host header. + /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets + pub fn from_request(req: &Request) -> Result { + let uri = req.uri(); + let path = uri.path(); + let query = uri.query(); + + let mut query = QueryParameters::from_query(query.unwrap_or_default())?; + + let res = router_match!(@gen_path_parser (req.method(), path, query) [ + OPTIONS _ => Options, + GET "/check" => CheckDomain, + GET "/health" => Health, + GET "/metrics" => Metrics, + GET "/v0/status" => GetClusterStatus, + GET "/v0/health" => GetClusterHealth, + POST "/v0/connect" => ConnectClusterNodes, + // Layout endpoints + GET "/v0/layout" => GetClusterLayout, + POST "/v0/layout" => UpdateClusterLayout, + POST "/v0/layout/apply" => ApplyClusterLayout, + POST "/v0/layout/revert" => RevertClusterLayout, + // API key endpoints + GET "/v0/key" if id => GetKeyInfo (query_opt::id, query_opt::search), + GET "/v0/key" if search => GetKeyInfo (query_opt::id, query_opt::search), + POST "/v0/key" if id => UpdateKey (query::id), + POST "/v0/key" => CreateKey, + POST "/v0/key/import" => ImportKey, + DELETE "/v0/key" if id => DeleteKey (query::id), + GET "/v0/key" => ListKeys, + // Bucket endpoints + GET "/v0/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v0/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v0/bucket" => ListBuckets, + POST "/v0/bucket" => CreateBucket, + DELETE "/v0/bucket" if id => DeleteBucket (query::id), + PUT "/v0/bucket" if id => UpdateBucket (query::id), + // Bucket-key permissions + POST "/v0/bucket/allow" => BucketAllowKey, + POST "/v0/bucket/deny" => BucketDenyKey, + // Bucket aliases + PUT "/v0/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), + DELETE "/v0/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), + PUT "/v0/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), + DELETE "/v0/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), + ]); + + if let Some(message) = query.nonempty_message() { + debug!("Unused query parameter: {}", message) + } + + Ok(res) + } +} + +generateQueryParameters! { + keywords: [], + fields: [ + "format" => format, + "id" => id, + "search" => search, + "globalAlias" => global_alias, + "alias" => alias, + "accessKeyId" => access_key_id + ] +} diff --git a/src/api/admin/router_v1.rs b/src/api/admin/router_v1.rs new file mode 100644 index 00000000..0b4901ea --- /dev/null +++ b/src/api/admin/router_v1.rs @@ -0,0 +1,236 @@ +use std::borrow::Cow; + +use hyper::{Method, Request}; + +use garage_api_common::router_macros::*; + +use crate::error::*; +use crate::router_v0; + +pub enum Authorization { + None, + MetricsToken, + AdminToken, +} + +router_match! {@func + +/// List of all Admin API endpoints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Endpoint { + Options, + CheckDomain, + Health, + Metrics, + GetClusterStatus, + GetClusterHealth, + ConnectClusterNodes, + // Layout + GetClusterLayout, + UpdateClusterLayout, + ApplyClusterLayout, + RevertClusterLayout, + // Keys + ListKeys, + CreateKey, + ImportKey, + GetKeyInfo { + id: Option, + search: Option, + show_secret_key: Option, + }, + DeleteKey { + id: String, + }, + UpdateKey { + id: String, + }, + // Buckets + ListBuckets, + CreateBucket, + GetBucketInfo { + id: Option, + global_alias: Option, + }, + DeleteBucket { + id: String, + }, + UpdateBucket { + id: String, + }, + // Bucket-Key Permissions + BucketAllowKey, + BucketDenyKey, + // Bucket aliases + GlobalAliasBucket { + id: String, + alias: String, + }, + GlobalUnaliasBucket { + id: String, + alias: String, + }, + LocalAliasBucket { + id: String, + access_key_id: String, + alias: String, + }, + LocalUnaliasBucket { + id: String, + access_key_id: String, + alias: String, + }, +}} + +impl Endpoint { + /// Determine which S3 endpoint a request is for using the request, and a bucket which was + /// possibly extracted from the Host header. + /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets + pub fn from_request(req: &Request) -> Result { + let uri = req.uri(); + let path = uri.path(); + let query = uri.query(); + + let mut query = QueryParameters::from_query(query.unwrap_or_default())?; + + let res = router_match!(@gen_path_parser (req.method(), path, query) [ + OPTIONS _ => Options, + GET "/check" => CheckDomain, + GET "/health" => Health, + GET "/metrics" => Metrics, + GET "/v1/status" => GetClusterStatus, + GET "/v1/health" => GetClusterHealth, + POST "/v1/connect" => ConnectClusterNodes, + // Layout endpoints + GET "/v1/layout" => GetClusterLayout, + POST "/v1/layout" => UpdateClusterLayout, + POST "/v1/layout/apply" => ApplyClusterLayout, + POST "/v1/layout/revert" => RevertClusterLayout, + // API key endpoints + GET "/v1/key" if id => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), + GET "/v1/key" if search => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), + POST "/v1/key" if id => UpdateKey (query::id), + POST "/v1/key" => CreateKey, + POST "/v1/key/import" => ImportKey, + DELETE "/v1/key" if id => DeleteKey (query::id), + GET "/v1/key" => ListKeys, + // Bucket endpoints + GET "/v1/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v1/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v1/bucket" => ListBuckets, + POST "/v1/bucket" => CreateBucket, + DELETE "/v1/bucket" if id => DeleteBucket (query::id), + PUT "/v1/bucket" if id => UpdateBucket (query::id), + // Bucket-key permissions + POST "/v1/bucket/allow" => BucketAllowKey, + POST "/v1/bucket/deny" => BucketDenyKey, + // Bucket aliases + PUT "/v1/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), + DELETE "/v1/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), + PUT "/v1/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), + DELETE "/v1/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), + ]); + + if let Some(message) = query.nonempty_message() { + debug!("Unused query parameter: {}", message) + } + + Ok(res) + } + /// Some endpoints work exactly the same in their v1/ version as they did in their v0/ version. + /// For these endpoints, we can convert a v0/ call to its equivalent as if it was made using + /// its v1/ URL. + pub fn from_v0(v0_endpoint: router_v0::Endpoint) -> Result { + match v0_endpoint { + // Cluster endpoints + router_v0::Endpoint::ConnectClusterNodes => Ok(Self::ConnectClusterNodes), + // - GetClusterStatus: response format changed + // - GetClusterHealth: response format changed + + // Layout endpoints + router_v0::Endpoint::RevertClusterLayout => Ok(Self::RevertClusterLayout), + // - GetClusterLayout: response format changed + // - UpdateClusterLayout: query format changed + // - ApplyCusterLayout: response format changed + + // Key endpoints + router_v0::Endpoint::ListKeys => Ok(Self::ListKeys), + router_v0::Endpoint::CreateKey => Ok(Self::CreateKey), + router_v0::Endpoint::GetKeyInfo { id, search } => Ok(Self::GetKeyInfo { + id, + search, + show_secret_key: Some("true".into()), + }), + router_v0::Endpoint::DeleteKey { id } => Ok(Self::DeleteKey { id }), + // - UpdateKey: response format changed (secret key no longer returned) + + // Bucket endpoints + router_v0::Endpoint::GetBucketInfo { id, global_alias } => { + Ok(Self::GetBucketInfo { id, global_alias }) + } + router_v0::Endpoint::ListBuckets => Ok(Self::ListBuckets), + router_v0::Endpoint::CreateBucket => Ok(Self::CreateBucket), + router_v0::Endpoint::DeleteBucket { id } => Ok(Self::DeleteBucket { id }), + router_v0::Endpoint::UpdateBucket { id } => Ok(Self::UpdateBucket { id }), + + // Bucket-key permissions + router_v0::Endpoint::BucketAllowKey => Ok(Self::BucketAllowKey), + router_v0::Endpoint::BucketDenyKey => Ok(Self::BucketDenyKey), + + // Bucket alias endpoints + router_v0::Endpoint::GlobalAliasBucket { id, alias } => { + Ok(Self::GlobalAliasBucket { id, alias }) + } + router_v0::Endpoint::GlobalUnaliasBucket { id, alias } => { + Ok(Self::GlobalUnaliasBucket { id, alias }) + } + router_v0::Endpoint::LocalAliasBucket { + id, + access_key_id, + alias, + } => Ok(Self::LocalAliasBucket { + id, + access_key_id, + alias, + }), + router_v0::Endpoint::LocalUnaliasBucket { + id, + access_key_id, + alias, + } => Ok(Self::LocalUnaliasBucket { + id, + access_key_id, + alias, + }), + + // For endpoints that have different body content syntax, issue + // deprecation warning + _ => Err(Error::bad_request(format!( + "v0/ endpoint is no longer supported: {}", + v0_endpoint.name() + ))), + } + } + /// Get the kind of authorization which is required to perform the operation. + pub fn authorization_type(&self) -> Authorization { + match self { + Self::Health => Authorization::None, + Self::CheckDomain => Authorization::None, + Self::Metrics => Authorization::MetricsToken, + _ => Authorization::AdminToken, + } + } +} + +generateQueryParameters! { + keywords: [], + fields: [ + "format" => format, + "id" => id, + "search" => search, + "globalAlias" => global_alias, + "alias" => alias, + "accessKeyId" => access_key_id, + "showSecretKey" => show_secret_key + ] +} diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml new file mode 100644 index 00000000..df01d59a --- /dev/null +++ b/src/api/common/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "garage_api_common" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Common functions for the API server crates for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_util.workspace = true + +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +thiserror.workspace = true +hex.workspace = true +hmac.workspace = true +md-5.workspace = true +tracing.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/common_error.rs b/src/api/common/common_error.rs similarity index 58% rename from src/api/common_error.rs rename to src/api/common/common_error.rs index 20f9f266..e596a6e9 100644 --- a/src/api/common_error.rs +++ b/src/api/common/common_error.rs @@ -1,61 +1,96 @@ -use err_derive::Error; +use std::convert::TryFrom; + use hyper::StatusCode; +use thiserror::Error; use garage_util::error::Error as GarageError; +use garage_model::helper::error::Error as HelperError; + /// Errors of this crate #[derive(Debug, Error)] pub enum CommonError { // ---- INTERNAL ERRORS ---- /// Error related to deeper parts of Garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), + #[error("Internal error: {0}")] + InternalError(#[from] GarageError), /// Error related to Hyper - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Internal error (Hyper error): {0}")] + Hyper(#[from] hyper::Error), /// Error related to HTTP - #[error(display = "Internal error (HTTP error): {}", _0)] - Http(#[error(source)] http::Error), + #[error("Internal error (HTTP error): {0}")] + Http(#[from] http::Error), // ---- GENERIC CLIENT ERRORS ---- /// Proper authentication was not provided - #[error(display = "Forbidden: {}", _0)] + #[error("Forbidden: {0}")] Forbidden(String), /// Generic bad request response with custom message - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), + /// The client sent a header with invalid value + #[error("Invalid header value: {0}")] + InvalidHeader(#[from] hyper::header::ToStrError), + // ---- SPECIFIC ERROR CONDITIONS ---- // These have to be error codes referenced in the S3 spec here: // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList /// The bucket requested don't exists - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] + #[error("Bucket already exists")] BucketAlreadyExists, /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] + #[error("Tried to delete a non-empty bucket")] BucketNotEmpty, // Category: bad request /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), } +#[macro_export] +macro_rules! commonErrorDerivative { + ( $error_struct: ident ) => { + impl From for $error_struct { + fn from(err: garage_util::error::Error) -> Self { + Self::Common(CommonError::InternalError(err)) + } + } + impl From for $error_struct { + fn from(err: http::Error) -> Self { + Self::Common(CommonError::Http(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::Error) -> Self { + Self::Common(CommonError::Hyper(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::header::ToStrError) -> Self { + Self::Common(CommonError::InvalidHeader(err)) + } + } + impl CommonErrorDerivative for $error_struct {} + }; +} + +pub use commonErrorDerivative; + impl CommonError { pub fn http_status_code(&self) -> StatusCode { match self { CommonError::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), ) => StatusCode::SERVICE_UNAVAILABLE, CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { StatusCode::INTERNAL_SERVER_ERROR @@ -64,7 +99,9 @@ impl CommonError { CommonError::Forbidden(_) => StatusCode::FORBIDDEN, CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, - CommonError::InvalidBucketName(_) => StatusCode::BAD_REQUEST, + CommonError::InvalidBucketName(_) | CommonError::InvalidHeader(_) => { + StatusCode::BAD_REQUEST + } } } @@ -72,9 +109,7 @@ impl CommonError { match self { CommonError::Forbidden(_) => "AccessDenied", CommonError::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), ) => "ServiceUnavailable", CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { "InternalError" @@ -84,6 +119,7 @@ impl CommonError { CommonError::BucketAlreadyExists => "BucketAlreadyExists", CommonError::BucketNotEmpty => "BucketNotEmpty", CommonError::InvalidBucketName(_) => "InvalidBucketName", + CommonError::InvalidHeader(_) => "InvalidHeaderValue", } } @@ -92,6 +128,39 @@ impl CommonError { } } +impl TryFrom for CommonError { + type Error = HelperError; + + fn try_from(err: HelperError) -> Result { + match err { + HelperError::Internal(i) => Ok(Self::InternalError(i)), + HelperError::BadRequest(b) => Ok(Self::BadRequest(b)), + HelperError::InvalidBucketName(n) => Ok(Self::InvalidBucketName(n)), + HelperError::NoSuchBucket(n) => Ok(Self::NoSuchBucket(n)), + e => Err(e), + } + } +} + +/// This function converts HelperErrors into CommonErrors, +/// for variants that exist in CommonError. +/// This is used for helper functions that might return InvalidBucketName +/// or NoSuchBucket for instance, and we want to pass that error +/// up to our caller. +pub fn pass_helper_error(err: HelperError) -> CommonError { + match CommonError::try_from(err) { + Ok(e) => e, + Err(e) => panic!("Helper error `{}` should hot have happenned here", e), + } +} + +pub fn helper_error_as_internal(err: HelperError) -> CommonError { + match err { + HelperError::Internal(e) => CommonError::InternalError(e), + e => CommonError::InternalError(GarageError::Message(e.to_string())), + } +} + pub trait CommonErrorDerivative: From { fn internal_error(msg: M) -> Self { Self::from(CommonError::InternalError(GarageError::Message( diff --git a/src/api/common/cors.rs b/src/api/common/cors.rs new file mode 100644 index 00000000..09b55c13 --- /dev/null +++ b/src/api/common/cors.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use http::header::{ + ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, + ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, +}; +use hyper::{body::Body, body::Incoming as IncomingBody, Request, Response, StatusCode}; + +use garage_model::bucket_table::{BucketParams, CorsRule as GarageCorsRule}; +use garage_model::garage::Garage; + +use crate::common_error::{ + helper_error_as_internal, CommonError, OkOrBadRequest, OkOrInternalError, +}; +use crate::helpers::*; + +pub fn find_matching_cors_rule<'a, B>( + bucket_params: &'a BucketParams, + req: &Request, +) -> Result, CommonError> { + if let Some(cors_config) = bucket_params.cors_config.get() { + if let Some(origin) = req.headers().get("Origin") { + let origin = origin.to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + return Ok(cors_config.iter().find(|rule| { + cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) + })); + } + } + Ok(None) +} + +pub fn cors_rule_matches<'a, HI, S>( + rule: &GarageCorsRule, + origin: &'a str, + method: &'a str, + mut request_headers: HI, +) -> bool +where + HI: Iterator, + S: AsRef, +{ + rule.allow_origins.iter().any(|x| x == "*" || x == origin) + && rule.allow_methods.iter().any(|x| x == "*" || x == method) + && request_headers.all(|h| { + rule.allow_headers + .iter() + .any(|x| x == "*" || x == h.as_ref()) + }) +} + +pub fn add_cors_headers( + resp: &mut Response, + rule: &GarageCorsRule, +) -> Result<(), http::header::InvalidHeaderValue> { + let h = resp.headers_mut(); + h.insert( + ACCESS_CONTROL_ALLOW_ORIGIN, + rule.allow_origins.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_METHODS, + rule.allow_methods.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_HEADERS, + rule.allow_headers.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_EXPOSE_HEADERS, + rule.expose_headers.join(", ").parse()?, + ); + Ok(()) +} + +pub async fn handle_options_api( + garage: Arc, + req: &Request, + bucket_name: Option, +) -> Result, CommonError> { + // FIXME: CORS rules of buckets with local aliases are + // not taken into account. + + // If the bucket name is a global bucket name, + // we try to apply the CORS rules of that bucket. + // If a user has a local bucket name that has + // the same name, its CORS rules won't be applied + // and will be shadowed by the rules of the globally + // existing bucket (but this is inevitable because + // OPTIONS calls are not auhtenticated). + if let Some(bn) = bucket_name { + let helper = garage.bucket_helper(); + let bucket_id = helper + .resolve_global_bucket_name(&bn) + .await + .map_err(helper_error_as_internal)?; + if let Some(id) = bucket_id { + let bucket = garage + .bucket_helper() + .get_existing_bucket(id) + .await + .map_err(helper_error_as_internal)?; + let bucket_params = bucket.state.into_option().unwrap(); + handle_options_for_bucket(req, &bucket_params) + } else { + // If there is a bucket name in the request, but that name + // does not correspond to a global alias for a bucket, + // then it's either a non-existing bucket or a local bucket. + // We have no way of knowing, because the request is not + // authenticated and thus we can't resolve local aliases. + // We take the permissive approach of allowing everything, + // because we don't want to prevent web apps that use + // local bucket names from making API calls. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "*") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } + } else { + // If there is no bucket name in the request, + // we are doing a ListBuckets call, which we want to allow + // for all origins. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } +} + +pub fn handle_options_for_bucket( + req: &Request, + bucket_params: &BucketParams, +) -> Result, CommonError> { + let origin = req + .headers() + .get("Origin") + .ok_or_bad_request("Missing Origin header")? + .to_str()?; + let request_method = req + .headers() + .get(ACCESS_CONTROL_REQUEST_METHOD) + .ok_or_bad_request("Missing Access-Control-Request-Method header")? + .to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + + if let Some(cors_config) = bucket_params.cors_config.get() { + let matching_rule = cors_config + .iter() + .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); + if let Some(rule) = matching_rule { + let mut resp = Response::builder() + .status(StatusCode::OK) + .body(EmptyBody::new())?; + add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; + return Ok(resp); + } + } + + Err(CommonError::Forbidden( + "This CORS request is not allowed.".into(), + )) +} diff --git a/src/api/encoding.rs b/src/api/common/encoding.rs similarity index 100% rename from src/api/encoding.rs rename to src/api/common/encoding.rs diff --git a/src/api/common/generic_server.rs b/src/api/common/generic_server.rs new file mode 100644 index 00000000..3f14c07d --- /dev/null +++ b/src/api/common/generic_server.rs @@ -0,0 +1,383 @@ +use std::convert::Infallible; +use std::fs::{self, Permissions}; +use std::os::unix::fs::PermissionsExt; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::Future; +use futures::stream::{futures_unordered::FuturesUnordered, StreamExt}; + +use http_body_util::BodyExt; +use hyper::header::HeaderValue; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{body::Incoming as IncomingBody, Request, Response}; +use hyper::{HeaderMap, StatusCode}; +use hyper_util::rt::TokioIo; + +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream}; +use tokio::sync::watch; +use tokio::time::{sleep_until, Instant}; + +use opentelemetry::{ + global, + metrics::{Counter, ValueRecorder}, + trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, + Context, KeyValue, +}; + +use garage_util::error::Error as GarageError; +use garage_util::forwarded_headers; +use garage_util::metrics::{gen_trace_id, RecordDuration}; +use garage_util::socket_address::UnixOrTCPSocketAddress; + +use crate::helpers::{BoxBody, ErrorBody}; + +pub trait ApiEndpoint: Send + Sync + 'static { + fn name(&self) -> &'static str; + fn add_span_attributes(&self, span: SpanRef<'_>); +} + +pub trait ApiError: std::error::Error + Send + Sync + 'static { + fn http_status_code(&self) -> StatusCode; + fn add_http_headers(&self, header_map: &mut HeaderMap); + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody; +} + +pub trait ApiHandler: Send + Sync + 'static { + const API_NAME: &'static str; + const API_NAME_DISPLAY: &'static str; + + type Endpoint: ApiEndpoint; + type Error: ApiError; + + fn parse_endpoint(&self, r: &Request) -> Result; + fn handle( + &self, + req: Request, + endpoint: Self::Endpoint, + ) -> impl Future>, Self::Error>> + Send; + + /// Returns the key id used to authenticate this request. The ID returned must be safe to + /// log. + fn key_id_from_request(&self, _req: &Request) -> Option { + None + } +} + +pub struct ApiServer { + region: String, + api_handler: A, + + // Metrics + request_counter: Counter, + error_counter: Counter, + request_duration: ValueRecorder, +} + +impl ApiServer { + pub fn new(region: String, api_handler: A) -> Arc { + let meter = global::meter("garage/api"); + Arc::new(Self { + region, + api_handler, + request_counter: meter + .u64_counter(format!("api.{}.request_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + error_counter: meter + .u64_counter(format!("api.{}.error_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints that resulted in errors", + A::API_NAME_DISPLAY + )) + .init(), + request_duration: meter + .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) + .with_description(format!( + "Duration of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + }) + } + + pub async fn run_server( + self: Arc, + bind_addr: UnixOrTCPSocketAddress, + unix_bind_addr_mode: Option, + must_exit: watch::Receiver, + ) -> Result<(), GarageError> { + let server_name = format!("{} API", A::API_NAME_DISPLAY); + info!("{} server listening on {}", server_name, bind_addr); + + match bind_addr { + UnixOrTCPSocketAddress::TCPSocket(addr) => { + let listener = TcpListener::bind(addr).await?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + UnixOrTCPSocketAddress::UnixSocket(ref path) => { + if path.exists() { + fs::remove_file(path)? + } + + let listener = UnixListener::bind(path)?; + let listener = UnixListenerOn(listener, path.display().to_string()); + + fs::set_permissions( + path, + Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)), + )?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + } + } + + async fn handler( + self: Arc, + req: Request, + addr: String, + ) -> Result>, http::Error> { + let uri = req.uri().clone(); + + let source = if let Ok(forwarded_for_ip_addr) = + forwarded_headers::handle_forwarded_for_headers(req.headers()) + { + format!("{forwarded_for_ip_addr} (via {addr})") + } else { + format!("{addr}") + }; + // we only do this to log the access key, so we can discard any error + let key = self + .api_handler + .key_id_from_request(&req) + .map(|k| format!("(key {k}) ")) + .unwrap_or_default(); + info!("{source} {key}{} {uri}", req.method()); + debug!("{:?}", req); + + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer + .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) + .with_trace_id(gen_trace_id()) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().to_string()), + ]) + .start(&tracer); + + let res = self + .handler_stage2(req) + .with_context(Context::current_with_span(span)) + .await; + + match res { + Ok(x) => { + debug!("{} {:?}", x.status(), x.headers()); + Ok(x) + } + Err(e) => { + let body = e.http_body(&self.region, uri.path()); + let mut http_error_builder = Response::builder().status(e.http_status_code()); + + if let Some(header_map) = http_error_builder.headers_mut() { + e.add_http_headers(header_map) + } + + let http_error = http_error_builder.body(body)?; + + if e.http_status_code().is_server_error() { + warn!("Response: error {}, {}", e.http_status_code(), e); + } else { + info!("Response: error {}, {}", e.http_status_code(), e); + } + Ok(http_error + .map(|body| BoxBody::new(body.map_err(|_: Infallible| unreachable!())))) + } + } + } + + async fn handler_stage2( + &self, + req: Request, + ) -> Result>, A::Error> { + let endpoint = self.api_handler.parse_endpoint(&req)?; + debug!("Endpoint: {}", endpoint.name()); + + let current_context = Context::current(); + let current_span = current_context.span(); + current_span.update_name::(format!( + "{} API {}", + A::API_NAME_DISPLAY, + endpoint.name() + )); + current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); + endpoint.add_span_attributes(current_span); + + let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; + + let res = self + .api_handler + .handle(req, endpoint) + .record_duration(&self.request_duration, &metrics_tags[..]) + .await; + + self.request_counter.add(1, &metrics_tags[..]); + + let status_code = match &res { + Ok(r) => r.status(), + Err(e) => e.http_status_code(), + }; + if status_code.is_client_error() || status_code.is_server_error() { + self.error_counter.add( + 1, + &[ + metrics_tags[0].clone(), + KeyValue::new("status_code", status_code.as_str().to_string()), + ], + ); + } + + res + } +} + +// ==== helper functions ==== + +pub trait Accept: Send + Sync + 'static { + type Stream: AsyncRead + AsyncWrite + Send + Sync + 'static; + fn accept(&self) -> impl Future> + Send; +} + +impl Accept for TcpListener { + type Stream = TcpStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.accept() + .await + .map(|(stream, addr)| (stream, addr.to_string())) + } +} + +pub struct UnixListenerOn(pub UnixListener, pub String); + +impl Accept for UnixListenerOn { + type Stream = UnixStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.0 + .accept() + .await + .map(|(stream, _addr)| (stream, self.1.clone())) + } +} + +pub async fn server_loop( + server_name: String, + listener: A, + handler: H, + mut must_exit: watch::Receiver, +) -> Result<(), GarageError> +where + A: Accept, + H: Fn(Request, String) -> F + Send + Sync + Clone + 'static, + F: Future>, http::Error>> + Send + 'static, + E: Send + Sync + std::error::Error + 'static, +{ + let (conn_in, mut conn_out) = tokio::sync::mpsc::unbounded_channel(); + let connection_collector = tokio::spawn({ + let server_name = server_name.clone(); + async move { + let mut connections = FuturesUnordered::>::new(); + loop { + let collect_next = async { + if connections.is_empty() { + futures::future::pending().await + } else { + connections.next().await + } + }; + tokio::select! { + result = collect_next => { + trace!("{} server: HTTP connection finished: {:?}", server_name, result); + } + new_fut = conn_out.recv() => { + match new_fut { + Some(f) => connections.push(f), + None => break, + } + } + } + } + let deadline = Instant::now() + Duration::from_secs(10); + while !connections.is_empty() { + info!( + "{} server: {} connections still open, deadline in {:.2}s", + server_name, + connections.len(), + (deadline - Instant::now()).as_secs_f32(), + ); + tokio::select! { + conn_res = connections.next() => { + trace!( + "{} server: HTTP connection finished: {:?}", + server_name, + conn_res.unwrap(), + ); + } + _ = sleep_until(deadline) => { + warn!("{} server: exit deadline reached with {} connections still open, killing them now", + server_name, + connections.len()); + for conn in connections.iter() { + conn.abort(); + } + for conn in connections { + assert!(conn.await.unwrap_err().is_cancelled()); + } + break; + } + } + } + } + }); + + while !*must_exit.borrow() { + let (stream, client_addr) = tokio::select! { + acc = listener.accept() => match acc { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue, + Err(e) => return Err(e.into()), + }, + _ = must_exit.changed() => continue, + }; + + let io = TokioIo::new(stream); + + let handler = handler.clone(); + let serve = move |req: Request| handler(req, client_addr.clone()); + + let fut = tokio::task::spawn(async move { + let io = Box::pin(io); + if let Err(e) = http1::Builder::new() + .serve_connection(io, service_fn(serve)) + .await + { + debug!("Error handling HTTP connection: {}", e); + } + }); + conn_in.send(fut)?; + } + + info!("{} server exiting", server_name); + drop(conn_in); + connection_collector.await?; + + Ok(()) +} diff --git a/src/api/helpers.rs b/src/api/common/helpers.rs similarity index 77% rename from src/api/helpers.rs rename to src/api/common/helpers.rs index 1d55ebd5..6fc4aa13 100644 --- a/src/api/helpers.rs +++ b/src/api/common/helpers.rs @@ -1,7 +1,21 @@ -use hyper::{Body, Request, Response}; -use idna::domain_to_unicode; +use std::convert::Infallible; +use std::sync::Arc; + +use futures::{Stream, StreamExt, TryStreamExt}; + +use http_body_util::{BodyExt, Full as FullBody}; +use hyper::{ + body::{Body, Bytes}, + Request, Response, +}; use serde::{Deserialize, Serialize}; +use garage_model::bucket_table::BucketParams; +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::Uuid; +use garage_util::error::Error as GarageError; + use crate::common_error::{CommonError as Error, *}; /// What kind of authorization is required to perform a given action @@ -17,6 +31,15 @@ pub enum Authorization { Owner, } +/// The values which are known for each request related to a bucket +pub struct ReqCtx { + pub garage: Arc, + pub bucket_id: Uuid, + pub bucket_name: String, + pub bucket_params: BucketParams, + pub api_key: Key, +} + /// Host to bucket /// /// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket", @@ -73,7 +96,7 @@ pub fn authority_to_host(authority: &str) -> Result { authority ))), }; - authority.map(|h| domain_to_unicode(h).0) + authority.map(|h| h.to_ascii_lowercase()) } /// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in @@ -138,18 +161,64 @@ pub fn key_after_prefix(pfx: &str) -> Option { None } -pub async fn parse_json_body Deserialize<'de>>(req: Request) -> Result { - let body = hyper::body::to_bytes(req.into_body()).await?; +// =============== body helpers ================= + +pub type EmptyBody = http_body_util::Empty; +pub type ErrorBody = FullBody; +pub type BoxBody = http_body_util::combinators::BoxBody; + +pub fn string_body(s: String) -> BoxBody { + bytes_body(bytes::Bytes::from(s.into_bytes())) +} +pub fn bytes_body(b: bytes::Bytes) -> BoxBody { + BoxBody::new(FullBody::new(b).map_err(|_: Infallible| unreachable!())) +} +pub fn empty_body() -> BoxBody { + BoxBody::new(http_body_util::Empty::new().map_err(|_: Infallible| unreachable!())) +} +pub fn error_body(s: String) -> ErrorBody { + ErrorBody::from(bytes::Bytes::from(s.into_bytes())) +} + +pub async fn parse_json_body(req: Request) -> Result +where + T: for<'de> Deserialize<'de>, + B: Body, + E: From<::Error> + From, +{ + let body = req.into_body().collect().await?.to_bytes(); let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; Ok(resp) } -pub fn json_ok_response(res: &T) -> Result, Error> { - let resp_json = serde_json::to_string_pretty(res).map_err(garage_util::error::Error::from)?; +pub fn json_ok_response(res: &T) -> Result>, E> +where + E: From, +{ + let resp_json = serde_json::to_string_pretty(res) + .map_err(GarageError::from) + .map_err(Error::from)?; Ok(Response::builder() .status(hyper::StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/json") - .body(Body::from(resp_json))?) + .body(string_body(resp_json)) + .unwrap()) +} + +pub fn body_stream(body: B) -> impl Stream> +where + B: Body, + ::Error: Into, + E: From, +{ + let stream = http_body_util::BodyStream::new(body); + let stream = TryStreamExt::map_err(stream, Into::into); + stream.map(|x| { + x.and_then(|f| { + f.into_data() + .map_err(|_| E::from(Error::bad_request("non-data frame"))) + }) + }) } pub fn is_default(v: &T) -> bool { @@ -293,9 +362,9 @@ mod tests { } #[derive(Serialize)] -pub(crate) struct CustomApiErrorBody { - pub(crate) code: String, - pub(crate) message: String, - pub(crate) region: String, - pub(crate) path: String, +pub struct CustomApiErrorBody { + pub code: String, + pub message: String, + pub region: String, + pub path: String, } diff --git a/src/api/common/lib.rs b/src/api/common/lib.rs new file mode 100644 index 00000000..0e655a53 --- /dev/null +++ b/src/api/common/lib.rs @@ -0,0 +1,12 @@ +//! Crate for serving a S3 compatible API +#[macro_use] +extern crate tracing; + +pub mod common_error; + +pub mod cors; +pub mod encoding; +pub mod generic_server; +pub mod helpers; +pub mod router_macros; +pub mod signature; diff --git a/src/api/router_macros.rs b/src/api/common/router_macros.rs similarity index 98% rename from src/api/router_macros.rs rename to src/api/common/router_macros.rs index cfecbc92..d9fe86db 100644 --- a/src/api/router_macros.rs +++ b/src/api/common/router_macros.rs @@ -1,5 +1,6 @@ /// This macro is used to generate very repetitive match {} blocks in this module /// It is _not_ made to be used anywhere else +#[macro_export] macro_rules! router_match { (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{ // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] } @@ -133,6 +134,7 @@ macro_rules! router_match { /// This macro is used to generate part of the code in this module. It must be called only one, and /// is useless outside of this module. +#[macro_export] macro_rules! generateQueryParameters { ( keywords: [ $($kw_param:expr => $kw_name: ident),* ], @@ -204,7 +206,7 @@ macro_rules! generateQueryParameters { } /// Get an error message in case not all parameters where used when extracting them to - /// build an Enpoint variant + /// build an Endpoint variant fn nonempty_message(&self) -> Option<&str> { if self.keyword.is_some() { Some("Keyword not used") @@ -220,5 +222,5 @@ macro_rules! generateQueryParameters { } } -pub(crate) use generateQueryParameters; -pub(crate) use router_match; +pub use generateQueryParameters; +pub use router_match; diff --git a/src/api/common/signature/body.rs b/src/api/common/signature/body.rs new file mode 100644 index 00000000..96be0d5b --- /dev/null +++ b/src/api/common/signature/body.rs @@ -0,0 +1,135 @@ +use std::sync::Mutex; + +use futures::prelude::*; +use futures::stream::BoxStream; +use http_body_util::{BodyExt, StreamBody}; +use hyper::body::{Bytes, Frame}; +use serde::Deserialize; +use tokio::sync::mpsc; +use tokio::task; + +use super::*; + +use crate::signature::checksum::*; + +pub struct ReqBody { + // why need mutex to be sync?? + pub(crate) stream: Mutex, Error>>>, + pub(crate) checksummer: Checksummer, + pub(crate) expected_checksums: ExpectedChecksums, + pub(crate) trailer_algorithm: Option, +} + +pub type StreamingChecksumReceiver = task::JoinHandle>; + +impl ReqBody { + pub fn add_expected_checksums(&mut self, more: ExpectedChecksums) { + if more.md5.is_some() { + self.expected_checksums.md5 = more.md5; + } + if more.sha256.is_some() { + self.expected_checksums.sha256 = more.sha256; + } + if more.extra.is_some() { + self.expected_checksums.extra = more.extra; + } + self.checksummer.add_expected(&self.expected_checksums); + } + + pub fn add_md5(&mut self) { + self.checksummer.add_md5(); + } + + // ============ non-streaming ============= + + pub async fn json Deserialize<'a>>(self) -> Result { + let body = self.collect().await?; + let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + Ok(resp) + } + + pub async fn collect(self) -> Result { + self.collect_with_checksums().await.map(|(b, _)| b) + } + + pub async fn collect_with_checksums(mut self) -> Result<(Bytes, Checksums), Error> { + let stream: BoxStream<_> = self.stream.into_inner().unwrap(); + let bytes = BodyExt::collect(StreamBody::new(stream)).await?.to_bytes(); + + self.checksummer.update(&bytes); + let checksums = self.checksummer.finalize(); + checksums.verify(&self.expected_checksums)?; + + Ok((bytes, checksums)) + } + + // ============ streaming ============= + + pub fn streaming_with_checksums( + self, + ) -> ( + BoxStream<'static, Result>, + StreamingChecksumReceiver, + ) { + let Self { + stream, + mut checksummer, + mut expected_checksums, + trailer_algorithm, + } = self; + + let (frame_tx, mut frame_rx) = mpsc::channel::>(5); + + let join_checksums = tokio::spawn(async move { + while let Some(frame) = frame_rx.recv().await { + match frame.into_data() { + Ok(data) => { + checksummer = tokio::task::spawn_blocking(move || { + checksummer.update(&data); + checksummer + }) + .await + .unwrap() + } + Err(frame) => { + let trailers = frame.into_trailers().unwrap(); + let algo = trailer_algorithm.unwrap(); + expected_checksums.extra = Some(extract_checksum_value(&trailers, algo)?); + break; + } + } + } + + if trailer_algorithm.is_some() && expected_checksums.extra.is_none() { + return Err(Error::bad_request("trailing checksum was not sent")); + } + + let checksums = checksummer.finalize(); + checksums.verify(&expected_checksums)?; + + Ok(checksums) + }); + + let stream: BoxStream<_> = stream.into_inner().unwrap(); + let stream = stream.filter_map(move |x| { + let frame_tx = frame_tx.clone(); + async move { + match x { + Err(e) => Some(Err(e)), + Ok(frame) => { + if frame.is_data() { + let data = frame.data_ref().unwrap().clone(); + let _ = frame_tx.send(frame).await; + Some(Ok(data)) + } else { + let _ = frame_tx.send(frame).await; + None + } + } + } + } + }); + + (stream.boxed(), join_checksums) + } +} diff --git a/src/api/common/signature/checksum.rs b/src/api/common/signature/checksum.rs new file mode 100644 index 00000000..3c5e7c53 --- /dev/null +++ b/src/api/common/signature/checksum.rs @@ -0,0 +1,323 @@ +use std::convert::{TryFrom, TryInto}; +use std::hash::Hasher; + +use base64::prelude::*; +use crc32c::Crc32cHasher as Crc32c; +use crc32fast::Hasher as Crc32; +use md5::{Digest, Md5}; +use sha1::Sha1; +use sha2::Sha256; + +use http::{HeaderMap, HeaderName, HeaderValue}; + +use garage_util::data::*; + +use super::*; + +pub use garage_model::s3::object_table::{ChecksumAlgorithm, ChecksumValue}; + +pub const CONTENT_MD5: HeaderName = HeaderName::from_static("content-md5"); + +pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-checksum-algorithm"); +pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode"); +pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32"); +pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c"); +pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1"); +pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256"); + +pub type Crc32Checksum = [u8; 4]; +pub type Crc32cChecksum = [u8; 4]; +pub type Md5Checksum = [u8; 16]; +pub type Sha1Checksum = [u8; 20]; +pub type Sha256Checksum = [u8; 32]; + +#[derive(Debug, Default, Clone)] +pub struct ExpectedChecksums { + // base64-encoded md5 (content-md5 header) + pub md5: Option, + // content_sha256 (as a Hash / FixedBytes32) + pub sha256: Option, + // extra x-amz-checksum-* header + pub extra: Option, +} + +pub struct Checksummer { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +#[derive(Default)] +pub struct Checksums { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +impl Checksummer { + pub fn new() -> Self { + Self { + crc32: None, + crc32c: None, + md5: None, + sha1: None, + sha256: None, + } + } + + pub fn init(expected: &ExpectedChecksums, add_md5: bool) -> Self { + let mut ret = Self::new(); + ret.add_expected(expected); + if add_md5 { + ret.add_md5(); + } + ret + } + + pub fn add_md5(&mut self) { + self.md5 = Some(Md5::new()); + } + + pub fn add_expected(&mut self, expected: &ExpectedChecksums) { + if expected.md5.is_some() { + self.md5 = Some(Md5::new()); + } + if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) { + self.sha256 = Some(Sha256::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) { + self.crc32 = Some(Crc32::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) { + self.crc32c = Some(Crc32c::default()); + } + if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) { + self.sha1 = Some(Sha1::new()); + } + } + + pub fn add(mut self, algo: Option) -> Self { + match algo { + Some(ChecksumAlgorithm::Crc32) => { + self.crc32 = Some(Crc32::new()); + } + Some(ChecksumAlgorithm::Crc32c) => { + self.crc32c = Some(Crc32c::default()); + } + Some(ChecksumAlgorithm::Sha1) => { + self.sha1 = Some(Sha1::new()); + } + Some(ChecksumAlgorithm::Sha256) => { + self.sha256 = Some(Sha256::new()); + } + None => (), + } + self + } + + pub fn update(&mut self, bytes: &[u8]) { + if let Some(crc32) = &mut self.crc32 { + crc32.update(bytes); + } + if let Some(crc32c) = &mut self.crc32c { + crc32c.write(bytes); + } + if let Some(md5) = &mut self.md5 { + md5.update(bytes); + } + if let Some(sha1) = &mut self.sha1 { + sha1.update(bytes); + } + if let Some(sha256) = &mut self.sha256 { + sha256.update(bytes); + } + } + + pub fn finalize(self) -> Checksums { + Checksums { + crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())), + crc32c: self + .crc32c + .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())), + md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()), + sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()), + sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()), + } + } +} + +impl Checksums { + pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> { + if let Some(expected_md5) = &expected.md5 { + match self.md5 { + Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (), + _ => { + return Err(Error::InvalidDigest( + "MD5 checksum verification failed (from content-md5)".into(), + )) + } + } + } + if let Some(expected_sha256) = &expected.sha256 { + match self.sha256 { + Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (), + _ => { + return Err(Error::InvalidDigest( + "SHA256 checksum verification failed (from x-amz-content-sha256)".into(), + )) + } + } + } + if let Some(extra) = expected.extra { + let algo = extra.algorithm(); + if self.extract(Some(algo)) != Some(extra) { + return Err(Error::InvalidDigest(format!( + "Failed to validate checksum for algorithm {:?}", + algo + ))); + } + } + Ok(()) + } + + pub fn extract(&self, algo: Option) -> Option { + match algo { + None => None, + Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())), + Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())), + Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())), + Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())), + } + } +} + +// ---- + +pub fn parse_checksum_algorithm(algo: &str) -> Result { + match algo { + "CRC32" => Ok(ChecksumAlgorithm::Crc32), + "CRC32C" => Ok(ChecksumAlgorithm::Crc32c), + "SHA1" => Ok(ChecksumAlgorithm::Sha1), + "SHA256" => Ok(ChecksumAlgorithm::Sha256), + _ => Err(Error::bad_request("invalid checksum algorithm")), + } +} + +/// Extract the value of the x-amz-checksum-algorithm header +pub fn request_checksum_algorithm( + headers: &HeaderMap, +) -> Result, Error> { + match headers.get(X_AMZ_CHECKSUM_ALGORITHM) { + None => Ok(None), + Some(x) => parse_checksum_algorithm(x.to_str()?).map(Some), + } +} + +pub fn request_trailer_checksum_algorithm( + headers: &HeaderMap, +) -> Result, Error> { + match headers.get(X_AMZ_TRAILER).map(|x| x.to_str()).transpose()? { + None => Ok(None), + Some(x) if x == X_AMZ_CHECKSUM_CRC32 => Ok(Some(ChecksumAlgorithm::Crc32)), + Some(x) if x == X_AMZ_CHECKSUM_CRC32C => Ok(Some(ChecksumAlgorithm::Crc32c)), + Some(x) if x == X_AMZ_CHECKSUM_SHA1 => Ok(Some(ChecksumAlgorithm::Sha1)), + Some(x) if x == X_AMZ_CHECKSUM_SHA256 => Ok(Some(ChecksumAlgorithm::Sha256)), + _ => Err(Error::bad_request("invalid checksum algorithm")), + } +} + +/// Extract the value of any of the x-amz-checksum-* headers +pub fn request_checksum_value( + headers: &HeaderMap, +) -> Result, Error> { + let mut ret = vec![]; + + if headers.contains_key(X_AMZ_CHECKSUM_CRC32) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_CRC32C) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32c)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_SHA1) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha1)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_SHA256) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha256)?); + } + + if ret.len() > 1 { + return Err(Error::bad_request( + "multiple x-amz-checksum-* headers given", + )); + } + Ok(ret.pop()) +} + +/// Checks for the presence of x-amz-checksum-algorithm +/// if so extract the corresponding x-amz-checksum-* value +pub fn extract_checksum_value( + headers: &HeaderMap, + algo: ChecksumAlgorithm, +) -> Result { + match algo { + ChecksumAlgorithm::Crc32 => { + let crc32 = headers + .get(X_AMZ_CHECKSUM_CRC32) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-crc32 header")?; + Ok(ChecksumValue::Crc32(crc32)) + } + ChecksumAlgorithm::Crc32c => { + let crc32c = headers + .get(X_AMZ_CHECKSUM_CRC32C) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-crc32c header")?; + Ok(ChecksumValue::Crc32c(crc32c)) + } + ChecksumAlgorithm::Sha1 => { + let sha1 = headers + .get(X_AMZ_CHECKSUM_SHA1) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-sha1 header")?; + Ok(ChecksumValue::Sha1(sha1)) + } + ChecksumAlgorithm::Sha256 => { + let sha256 = headers + .get(X_AMZ_CHECKSUM_SHA256) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-sha256 header")?; + Ok(ChecksumValue::Sha256(sha256)) + } + } +} + +pub fn add_checksum_response_headers( + checksum: &Option, + mut resp: http::response::Builder, +) -> http::response::Builder { + match checksum { + Some(ChecksumValue::Crc32(crc32)) => { + resp = resp.header(X_AMZ_CHECKSUM_CRC32, BASE64_STANDARD.encode(&crc32)); + } + Some(ChecksumValue::Crc32c(crc32c)) => { + resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c)); + } + Some(ChecksumValue::Sha1(sha1)) => { + resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1)); + } + Some(ChecksumValue::Sha256(sha256)) => { + resp = resp.header(X_AMZ_CHECKSUM_SHA256, BASE64_STANDARD.encode(&sha256)); + } + None => (), + } + resp +} diff --git a/src/api/signature/error.rs b/src/api/common/signature/error.rs similarity index 60% rename from src/api/signature/error.rs rename to src/api/common/signature/error.rs index f0d7c816..a1b353e1 100644 --- a/src/api/signature/error.rs +++ b/src/api/common/signature/error.rs @@ -1,4 +1,4 @@ -use err_derive::Error; +use thiserror::Error; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; @@ -6,22 +6,22 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error Common(CommonError), /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), } impl From for Error diff --git a/src/api/common/signature/mod.rs b/src/api/common/signature/mod.rs new file mode 100644 index 00000000..50fbd304 --- /dev/null +++ b/src/api/common/signature/mod.rs @@ -0,0 +1,118 @@ +use chrono::{DateTime, Utc}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use hyper::header::HeaderName; +use hyper::{body::Incoming as IncomingBody, Request}; + +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::{sha256sum, Hash}; + +use error::*; + +pub mod body; +pub mod checksum; +pub mod error; +pub mod payload; +pub mod streaming; + +pub const SHORT_DATE: &str = "%Y%m%d"; +pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; + +// ---- Constants used in AWSv4 signatures ---- + +pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm"); +pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential"); +pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date"); +pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires"); +pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders"); +pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature"); +pub const X_AMZ_CONTENT_SHA256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); +pub const X_AMZ_TRAILER: HeaderName = HeaderName::from_static("x-amz-trailer"); + +/// Result of `sha256("")` +pub(crate) const EMPTY_STRING_HEX_DIGEST: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +// Signature calculation algorithm +pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; +type HmacSha256 = Hmac; + +// Possible values for x-amz-content-sha256, in addition to the actual sha256 +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const STREAMING_UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; +pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + +// Used in the computation of StringToSign +pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD"; + +// ---- enums to describe stuff going on in signature calculation ---- + +#[derive(Debug)] +pub enum ContentSha256Header { + UnsignedPayload, + Sha256Checksum(Hash), + StreamingPayload { trailer: bool, signed: bool }, +} + +// ---- top-level functions ---- + +pub struct VerifiedRequest { + pub request: Request, + pub access_key: Key, + pub content_sha256_header: ContentSha256Header, +} + +pub async fn verify_request( + garage: &Garage, + mut req: Request, + service: &'static str, +) -> Result { + let checked_signature = payload::check_payload_signature(&garage, &mut req, service).await?; + + let request = streaming::parse_streaming_body( + req, + &checked_signature, + &garage.config.s3_api.s3_region, + service, + )?; + + let access_key = checked_signature + .key + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; + + Ok(VerifiedRequest { + request, + access_key, + content_sha256_header: checked_signature.content_sha256_header, + }) +} + +pub fn signing_hmac( + datetime: &DateTime, + secret_key: &str, + region: &str, + service: &str, +) -> Result { + let secret = String::from("AWS4") + secret_key; + let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; + date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); + let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; + region_hmac.update(region.as_bytes()); + let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; + service_hmac.update(service.as_bytes()); + let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; + signing_hmac.update(b"aws4_request"); + let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; + Ok(hmac) +} + +pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { + format!( + "{}/{}/{}/aws4_request", + datetime.format(SHORT_DATE), + region, + service + ) +} diff --git a/src/api/common/signature/payload.rs b/src/api/common/signature/payload.rs new file mode 100644 index 00000000..3939da19 --- /dev/null +++ b/src/api/common/signature/payload.rs @@ -0,0 +1,584 @@ +use std::collections::HashMap; +use std::convert::TryFrom; + +use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; +use hmac::Mac; +use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST}; +use hyper::{body::Incoming as IncomingBody, Method, Request}; +use sha2::{Digest, Sha256}; + +use garage_table::*; +use garage_util::data::Hash; + +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use super::*; + +use crate::encoding::uri_encode; + +pub type QueryMap = HeaderMap; +pub struct QueryValue { + /// Original key with potential uppercase characters, + /// for use in signature calculation + key: String, + value: String, +} + +#[derive(Debug)] +pub struct CheckedSignature { + pub key: Option, + pub content_sha256_header: ContentSha256Header, + pub signature_header: Option, +} + +pub async fn check_payload_signature( + garage: &Garage, + request: &mut Request, + service: &'static str, +) -> Result { + let query = parse_query_map(request.uri())?; + + if query.contains_key(&X_AMZ_ALGORITHM) { + // We check for presigned-URL-style authentication first, because + // the browser or something else could inject an Authorization header + // that is totally unrelated to AWS signatures. + check_presigned_signature(garage, service, request, query).await + } else if request.headers().contains_key(AUTHORIZATION) { + check_standard_signature(garage, service, request, query).await + } else { + // Unsigned (anonymous) request + let content_sha256 = request + .headers() + .get(X_AMZ_CONTENT_SHA256) + .map(|x| x.to_str()) + .transpose()?; + Ok(CheckedSignature { + key: None, + content_sha256_header: parse_x_amz_content_sha256(content_sha256)?, + signature_header: None, + }) + } +} + +fn parse_x_amz_content_sha256(header: Option<&str>) -> Result { + let header = match header { + Some(x) => x, + None => return Ok(ContentSha256Header::UnsignedPayload), + }; + if header == UNSIGNED_PAYLOAD { + Ok(ContentSha256Header::UnsignedPayload) + } else if let Some(rest) = header.strip_prefix("STREAMING-") { + let (trailer, algo) = if let Some(rest2) = rest.strip_suffix("-TRAILER") { + (true, rest2) + } else { + (false, rest) + }; + let signed = match algo { + AWS4_HMAC_SHA256_PAYLOAD => true, + UNSIGNED_PAYLOAD => false, + _ => { + return Err(Error::bad_request( + "invalid or unsupported x-amz-content-sha256", + )) + } + }; + Ok(ContentSha256Header::StreamingPayload { trailer, signed }) + } else { + let sha256 = hex::decode(header) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid content sha256 hash")?; + Ok(ContentSha256Header::Sha256Checksum(sha256)) + } +} + +async fn check_standard_signature( + garage: &Garage, + service: &'static str, + request: &Request, + query: QueryMap, +) -> Result { + let authorization = Authorization::parse_header(request.headers())?; + + // Verify that all necessary request headers are included in signed_headers + // The following must be included for all signatures: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request (except x-amz-content-sha256) + // AWS also indicates that the Content-Type header should be signed if + // it is used, but Minio client doesn't sign it so we don't check it for compatibility. + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request:\n{}", canonical_request); + trace!("string to sign:\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + let content_sha256_header = parse_x_amz_content_sha256(Some(&authorization.content_sha256))?; + + Ok(CheckedSignature { + key: Some(key), + content_sha256_header, + signature_header: Some(authorization.signature), + }) +} + +async fn check_presigned_signature( + garage: &Garage, + service: &'static str, + request: &mut Request, + mut query: QueryMap, +) -> Result { + let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap(); + let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; + + // Verify that all necessary request headers are included in signed_headers + // For AWSv4 pre-signed URLs, the following must be included: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request (except x-amz-content-sha256) + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + // The X-Amz-Signature value is passed as a query parameter, + // but the signature cannot be computed from a string that contains itself. + // AWS specifies that all query params except X-Amz-Signature are included + // in the canonical request. + query.remove(&X_AMZ_SIGNATURE); + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request (presigned url):\n{}", canonical_request); + trace!("string to sign (presigned url):\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + // In the page on presigned URLs, AWS specifies that if a signed query + // parameter and a signed header of the same name have different values, + // then an InvalidRequest error is raised. + let headers_mut = request.headers_mut(); + for (name, value) in query.iter() { + if let Some(existing) = headers_mut.get(name) { + if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { + return Err(Error::bad_request(format!( + "Conflicting values for `{}` in query parameters and request headers", + name + ))); + } + } + if name.as_str().starts_with("x-amz-") { + // Query parameters that start by x-amz- are actually intended to stand in for + // headers that can't be added at the time the request is made. + // What we do is just add them to the Request object as regular headers, + // that will be handled downstream as if they were included like in a normal request. + // (Here we allow such query parameters to override headers with the same name + // that are not signed, however there is not much reason that this would happen) + headers_mut.insert( + name, + HeaderValue::from_bytes(value.value.as_bytes()) + .ok_or_bad_request("invalid query parameter value")?, + ); + } + } + + // Presigned URLs always use UNSIGNED-PAYLOAD, + // so there is no sha256 hash to return. + Ok(CheckedSignature { + key: Some(key), + content_sha256_header: ContentSha256Header::UnsignedPayload, + signature_header: Some(authorization.signature), + }) +} + +pub fn parse_query_map(uri: &http::uri::Uri) -> Result { + let mut query = QueryMap::with_capacity(0); + if let Some(query_str) = uri.query() { + let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); + for (key, val) in query_pairs { + let name = + HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?; + + let value = QueryValue { + key: key.to_string(), + value: val.into_owned(), + }; + + if query.insert(name, value).is_some() { + return Err(Error::bad_request(format!( + "duplicate query parameter: `{}`", + key + ))); + } + } + } + Ok(query) +} + +fn parse_credential(cred: &str) -> Result<(String, String), Error> { + let first_slash = cred + .find('/') + .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; + let (key_id, scope) = cred.split_at(first_slash); + Ok(( + key_id.to_string(), + scope.trim_start_matches('/').to_string(), + )) +} + +fn split_signed_headers(authorization: &Authorization) -> Result, Error> { + let mut signed_headers = authorization + .signed_headers + .split(';') + .map(HeaderName::try_from) + .collect::, _>>() + .ok_or_bad_request("invalid header name")?; + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + Ok(signed_headers) +} + +fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> { + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } + for (name, _) in headers.iter() { + // Enforce signature of all x-amz-* headers, except x-amz-content-sh256 + // because it is included in the canonical request in all cases + if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } + } + Ok(()) +} + +pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { + let mut hasher = Sha256::default(); + hasher.update(canonical_req.as_bytes()); + [ + AWS4_HMAC_SHA256, + &datetime.format(LONG_DATETIME).to_string(), + scope_string, + &hex::encode(hasher.finalize().as_slice()), + ] + .join("\n") +} + +pub fn canonical_request( + service: &'static str, + method: &Method, + canonical_uri: &str, + query: &QueryMap, + headers: &HeaderMap, + signed_headers: &[HeaderName], + content_sha256: &str, +) -> Result { + // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded + // a second time when building the canonical request, as specified in this documentation page: + // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html + // However this documentation page is for a specific service ("roles anywhere"), and + // in the S3 service we know for a fact that there is no double-urlencoding, because all of + // the tests we made with external software work without it. + // + // The theory is that double-urlencoding occurs for all services except S3, + // which is what is implemented in rusoto_signature: + // -> https://docs.rs/rusoto_signature/latest/src/rusoto_signature/signature.rs.html#464 + // + // Digging into the code of the official AWS Rust SDK, we learn that double-URI-encoding can + // be set or unset on a per-request basis (the signature crates, aws-sigv4 and aws-sig-auth, + // are agnostic to this). Grepping the codebase confirms that S3 is the only API for which + // double_uri_encode is set to false, meaning it is true (its default value) for all other + // AWS services. We will therefore implement this behavior in Garage as well. + // + // Note that this documentation page, which is touted as the "authoritative reference" on + // AWSv4 signatures, makes no mention of either single- or double-urlencoding: + // -> https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html + // This page of the S3 documentation does also not mention anything specific: + // -> https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // + // Note that there is also the issue of path normalization, which I hope is unrelated to the + // one of URI-encoding. At least in aws-sigv4 both parameters can be set independently, + // and rusoto_signature does not seem to do any effective path normalization, even though + // it mentions it in the comments (same link to the source code as above). + // We make the explicit choice of NOT normalizing paths in the K2V API because doing so + // would make non-normalized paths invalid K2V partition keys, and we don't want that. + let canonical_uri: std::borrow::Cow = if service != "s3" { + uri_encode(canonical_uri, false).into() + } else { + canonical_uri.into() + }; + + // Canonical query string from passed HeaderMap + let canonical_query_string = { + let mut items = Vec::with_capacity(query.len()); + for (_, QueryValue { key, value }) in query.iter() { + items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); + } + items.sort(); + items.join("&") + }; + + // Canonical header string calculated from signed headers + let canonical_header_string = signed_headers + .iter() + .map(|name| { + let value = headers + .get(name) + .ok_or_bad_request(format!("signed header `{}` is not present", name))?; + let value = std::str::from_utf8(value.as_bytes())?; + Ok(format!("{}:{}", name.as_str(), value.trim())) + }) + .collect::, Error>>()? + .join("\n"); + let signed_headers = signed_headers.join(";"); + + let list = [ + method.as_str(), + &canonical_uri, + &canonical_query_string, + &canonical_header_string, + "", + &signed_headers, + content_sha256, + ]; + Ok(list.join("\n")) +} + +pub fn parse_date(date: &str) -> Result, Error> { + let date: NaiveDateTime = + NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; + Ok(Utc.from_utc_datetime(&date)) +} + +pub async fn verify_v4( + garage: &Garage, + service: &str, + auth: &Authorization, + payload: &[u8], +) -> Result { + let scope_expected = compute_scope(&auth.date, &garage.config.s3_api.s3_region, service); + if auth.scope != scope_expected { + return Err(Error::AuthorizationHeaderMalformed(auth.scope.to_string())); + } + + let key = garage + .key_table + .get(&EmptyKey, &auth.key_id) + .await? + .filter(|k| !k.state.is_deleted()) + .ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?; + let key_p = key.params().unwrap(); + + let mut hmac = signing_hmac( + &auth.date, + &key_p.secret_key, + &garage.config.s3_api.s3_region, + service, + ) + .ok_or_internal_error("Unable to build signing HMAC")?; + hmac.update(payload); + let signature = + hex::decode(&auth.signature).map_err(|_| Error::forbidden("Invalid signature"))?; + if hmac.verify_slice(&signature).is_err() { + return Err(Error::forbidden("Invalid signature")); + } + + Ok(key) +} + +// ============ Authorization header, or X-Amz-* query params ========= + +pub struct Authorization { + pub key_id: String, + scope: String, + signed_headers: String, + signature: String, + content_sha256: String, + date: DateTime, +} + +impl Authorization { + pub fn parse_header(headers: &HeaderMap) -> Result { + let authorization = headers + .get(AUTHORIZATION) + .ok_or_bad_request("Missing authorization header")? + .to_str()?; + + let (auth_kind, rest) = authorization + .split_once(' ') + .ok_or_bad_request("Authorization field to short")?; + + if auth_kind != AWS4_HMAC_SHA256 { + return Err(Error::bad_request("Unsupported authorization method")); + } + + let mut auth_params = HashMap::new(); + for auth_part in rest.split(',') { + let auth_part = auth_part.trim(); + let eq = auth_part + .find('=') + .ok_or_bad_request("Field without value in authorization header")?; + let (key, value) = auth_part.split_at(eq); + auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); + } + + let cred = auth_params + .get("Credential") + .ok_or_bad_request("Could not find Credential in Authorization field")?; + let signed_headers = auth_params + .get("SignedHeaders") + .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? + .to_string(); + let signature = auth_params + .get("Signature") + .ok_or_bad_request("Could not find Signature in Authorization field")? + .to_string(); + + let content_sha256 = headers + .get(X_AMZ_CONTENT_SHA256) + .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; + + let date = headers + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(cred)?; + let auth = Authorization { + key_id, + scope, + signed_headers, + signature, + content_sha256: content_sha256.to_str()?.to_string(), + date, + }; + Ok(auth) + } + + fn parse_presigned(algorithm: &str, query: &QueryMap) -> Result { + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let cred = query + .get(&X_AMZ_CREDENTIAL) + .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; + let signed_headers = query + .get(&X_AMZ_SIGNEDHEADERS) + .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; + let signature = query + .get(&X_AMZ_SIGNATURE) + .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; + + let duration = query + .get(&X_AMZ_EXPIRES) + .ok_or_bad_request("X-Amz-Expires not found in query parameters")? + .value + .parse() + .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; + + if duration > 7 * 24 * 3600 { + return Err(Error::bad_request( + "X-Amz-Expires may not exceed a week".to_string(), + )); + } + + let date = query + .get(&X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")?; + let date = parse_date(&date.value)?; + + if Utc::now() - date > Duration::seconds(duration) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(&cred.value)?; + Ok(Authorization { + key_id, + scope, + signed_headers: signed_headers.value.clone(), + signature: signature.value.clone(), + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }) + } + + pub fn parse_form(params: &HeaderMap) -> Result { + let algorithm = params + .get(X_AMZ_ALGORITHM) + .ok_or_bad_request("Missing X-Amz-Algorithm header")? + .to_str()?; + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let credential = params + .get(X_AMZ_CREDENTIAL) + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? + .to_str()?; + let signature = params + .get(X_AMZ_SIGNATURE) + .ok_or_bad_request("No signature was provided")? + .to_str()? + .to_string(); + let date = params + .get(X_AMZ_DATE) + .ok_or_bad_request("No date was provided")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(credential)?; + let auth = Authorization { + key_id, + scope, + signed_headers: "".to_string(), + signature, + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }; + Ok(auth) + } +} diff --git a/src/api/common/signature/streaming.rs b/src/api/common/signature/streaming.rs new file mode 100644 index 00000000..64362727 --- /dev/null +++ b/src/api/common/signature/streaming.rs @@ -0,0 +1,618 @@ +use std::pin::Pin; +use std::sync::Mutex; + +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; +use futures::prelude::*; +use futures::task; +use hmac::Mac; +use http::header::{HeaderMap, HeaderValue, CONTENT_ENCODING}; +use hyper::body::{Bytes, Frame, Incoming as IncomingBody}; +use hyper::Request; + +use garage_util::data::Hash; + +use super::*; + +use crate::helpers::body_stream; +use crate::signature::checksum::*; +use crate::signature::payload::CheckedSignature; + +pub use crate::signature::body::ReqBody; + +pub fn parse_streaming_body( + mut req: Request, + checked_signature: &CheckedSignature, + region: &str, + service: &str, +) -> Result, Error> { + debug!( + "Content signature mode: {:?}", + checked_signature.content_sha256_header + ); + + match checked_signature.content_sha256_header { + ContentSha256Header::StreamingPayload { signed, trailer } => { + // Sanity checks + if !signed && !trailer { + return Err(Error::bad_request( + "STREAMING-UNSIGNED-PAYLOAD without trailer is not a valid combination", + )); + } + + // Remove the aws-chunked component in the content-encoding: header + // Note: this header is not properly sent by minio client, so don't fail + // if it is absent from the request. + if let Some(content_encoding) = req.headers_mut().remove(CONTENT_ENCODING) { + if let Some(rest) = content_encoding.as_bytes().strip_prefix(b"aws-chunked,") { + req.headers_mut() + .insert(CONTENT_ENCODING, HeaderValue::from_bytes(rest).unwrap()); + } else if content_encoding != "aws-chunked" { + return Err(Error::bad_request( + "content-encoding does not contain aws-chunked for STREAMING-*-PAYLOAD", + )); + } + } + + // If trailer header is announced, add the calculation of the requested checksum + let mut checksummer = Checksummer::init(&Default::default(), false); + let trailer_algorithm = if trailer { + let algo = Some( + request_trailer_checksum_algorithm(req.headers())? + .ok_or_bad_request("Missing x-amz-trailer header")?, + ); + checksummer = checksummer.add(algo); + algo + } else { + None + }; + + // For signed variants, determine signing parameters + let sign_params = if signed { + let signature = checked_signature + .signature_header + .clone() + .ok_or_bad_request("No signature provided")?; + let signature = hex::decode(signature) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid signature")?; + + let secret_key = checked_signature + .key + .as_ref() + .ok_or_bad_request("Cannot sign streaming payload without signing key")? + .state + .as_option() + .ok_or_internal_error("Deleted key state")? + .secret_key + .to_string(); + + let date = req + .headers() + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) + .ok_or_bad_request("Invalid date")?; + let date: DateTime = Utc.from_utc_datetime(&date); + + let scope = compute_scope(&date, region, service); + let signing_hmac = + crate::signature::signing_hmac(&date, &secret_key, region, service) + .ok_or_internal_error("Unable to build signing HMAC")?; + + Some(SignParams { + datetime: date, + scope, + signing_hmac, + previous_signature: signature, + }) + } else { + None + }; + + Ok(req.map(move |body| { + let stream = body_stream::<_, Error>(body); + + let signed_payload_stream = + StreamingPayloadStream::new(stream, sign_params, trailer).map_err(Error::from); + ReqBody { + stream: Mutex::new(signed_payload_stream.boxed()), + checksummer, + expected_checksums: Default::default(), + trailer_algorithm, + } + })) + } + _ => Ok(req.map(|body| { + let expected_checksums = ExpectedChecksums { + sha256: match &checked_signature.content_sha256_header { + ContentSha256Header::Sha256Checksum(sha256) => Some(*sha256), + _ => None, + }, + ..Default::default() + }; + let checksummer = Checksummer::init(&expected_checksums, false); + + let stream = http_body_util::BodyStream::new(body).map_err(Error::from); + ReqBody { + stream: Mutex::new(stream.boxed()), + checksummer, + expected_checksums, + trailer_algorithm: None, + } + })), + } +} + +fn compute_streaming_payload_signature( + signing_hmac: &HmacSha256, + date: DateTime, + scope: &str, + previous_signature: Hash, + content_sha256: Hash, +) -> Result { + let string_to_sign = [ + AWS4_HMAC_SHA256_PAYLOAD, + &date.format(LONG_DATETIME).to_string(), + scope, + &hex::encode(previous_signature), + EMPTY_STRING_HEX_DIGEST, + &hex::encode(content_sha256), + ] + .join("\n"); + + let mut hmac = signing_hmac.clone(); + hmac.update(string_to_sign.as_bytes()); + + Hash::try_from(&hmac.finalize().into_bytes()) + .ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into())) +} + +fn compute_streaming_trailer_signature( + signing_hmac: &HmacSha256, + date: DateTime, + scope: &str, + previous_signature: Hash, + trailer_sha256: Hash, +) -> Result { + let string_to_sign = [ + AWS4_HMAC_SHA256_PAYLOAD, + &date.format(LONG_DATETIME).to_string(), + scope, + &hex::encode(previous_signature), + &hex::encode(trailer_sha256), + ] + .join("\n"); + + let mut hmac = signing_hmac.clone(); + hmac.update(string_to_sign.as_bytes()); + + Hash::try_from(&hmac.finalize().into_bytes()) + .ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into())) +} + +mod payload { + use http::{HeaderName, HeaderValue}; + + use garage_util::data::Hash; + + use nom::bytes::streaming::{tag, take_while}; + use nom::character::streaming::hex_digit1; + use nom::combinator::{map_res, opt}; + use nom::number::streaming::hex_u32; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(|e| e.map(Error::Parser))? + }; + } + + pub enum Error { + Parser(nom::error::Error), + BadSignature, + } + + impl Error { + pub fn description(&self) -> &str { + match *self { + Error::Parser(ref e) => e.code.description(), + Error::BadSignature => "Bad signature", + } + } + } + + #[derive(Debug, Clone)] + pub struct ChunkHeader { + pub size: usize, + pub signature: Option, + } + + impl ChunkHeader { + pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, size) = try_parse!(hex_u32(input)); + let (input, _) = try_parse!(tag(";")(input)); + + let (input, _) = try_parse!(tag("chunk-signature=")(input)); + let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); + let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; + + let (input, _) = try_parse!(tag("\r\n")(input)); + + let header = ChunkHeader { + size: size as usize, + signature: Some(signature), + }; + + Ok((input, header)) + } + + pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, size) = try_parse!(hex_u32(input)); + let (input, _) = try_parse!(tag("\r\n")(input)); + + let header = ChunkHeader { + size: size as usize, + signature: None, + }; + + Ok((input, header)) + } + } + + #[derive(Debug, Clone)] + pub struct TrailerChunk { + pub header_name: HeaderName, + pub header_value: HeaderValue, + pub signature: Option, + } + + impl TrailerChunk { + fn parse_content(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, header_name) = try_parse!(map_res( + take_while(|c: u8| c.is_ascii_alphanumeric() || c == b'-'), + HeaderName::from_bytes + )(input)); + let (input, _) = try_parse!(tag(b":")(input)); + let (input, header_value) = try_parse!(map_res( + take_while(|c: u8| c.is_ascii_alphanumeric() || b"+/=".contains(&c)), + HeaderValue::from_bytes + )(input)); + + // Possible '\n' after the header value, depends on clients + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + let (input, _) = try_parse!(opt(tag(b"\n"))(input)); + + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok(( + input, + TrailerChunk { + header_name, + header_value, + signature: None, + }, + )) + } + pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, trailer) = Self::parse_content(input)?; + + let (input, _) = try_parse!(tag(b"x-amz-trailer-signature:")(input)); + let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); + let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok(( + input, + TrailerChunk { + signature: Some(signature), + ..trailer + }, + )) + } + pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, trailer) = Self::parse_content(input)?; + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok((input, trailer)) + } + } +} + +#[derive(Debug)] +pub enum StreamingPayloadError { + Stream(Error), + InvalidSignature, + Message(String), +} + +impl StreamingPayloadError { + fn message(msg: &str) -> Self { + StreamingPayloadError::Message(msg.into()) + } +} + +impl From for Error { + fn from(err: StreamingPayloadError) -> Self { + match err { + StreamingPayloadError::Stream(e) => e, + StreamingPayloadError::InvalidSignature => { + Error::bad_request("Invalid payload signature") + } + StreamingPayloadError::Message(e) => { + Error::bad_request(format!("Chunk format error: {}", e)) + } + } + } +} + +impl From> for StreamingPayloadError { + fn from(err: payload::Error) -> Self { + Self::message(err.description()) + } +} + +impl From> for StreamingPayloadError { + fn from(err: nom::error::Error) -> Self { + Self::message(err.code.description()) + } +} + +enum StreamingPayloadChunk { + Chunk { + header: payload::ChunkHeader, + data: Bytes, + }, + Trailer(payload::TrailerChunk), +} + +struct SignParams { + datetime: DateTime, + scope: String, + signing_hmac: HmacSha256, + previous_signature: Hash, +} + +#[pin_project::pin_project] +pub struct StreamingPayloadStream +where + S: Stream>, +{ + #[pin] + stream: S, + buf: bytes::BytesMut, + signing: Option, + has_trailer: bool, + done: bool, +} + +impl StreamingPayloadStream +where + S: Stream>, +{ + fn new(stream: S, signing: Option, has_trailer: bool) -> Self { + Self { + stream, + buf: bytes::BytesMut::new(), + signing, + has_trailer, + done: false, + } + } + + fn parse_next( + input: &[u8], + is_signed: bool, + has_trailer: bool, + ) -> nom::IResult<&[u8], StreamingPayloadChunk, StreamingPayloadError> { + use nom::bytes::streaming::{tag, take}; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(nom::Err::convert)? + }; + } + + let (input, header) = if is_signed { + try_parse!(payload::ChunkHeader::parse_signed(input)) + } else { + try_parse!(payload::ChunkHeader::parse_unsigned(input)) + }; + + // 0-sized chunk is the last + if header.size == 0 { + if has_trailer { + let (input, trailer) = if is_signed { + try_parse!(payload::TrailerChunk::parse_signed(input)) + } else { + try_parse!(payload::TrailerChunk::parse_unsigned(input)) + }; + return Ok((input, StreamingPayloadChunk::Trailer(trailer))); + } else { + return Ok(( + input, + StreamingPayloadChunk::Chunk { + header, + data: Bytes::new(), + }, + )); + } + } + + let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); + let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); + + let data = Bytes::from(data.to_vec()); + + Ok((input, StreamingPayloadChunk::Chunk { header, data })) + } +} + +impl Stream for StreamingPayloadStream +where + S: Stream> + Unpin, +{ + type Item = Result, StreamingPayloadError>; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll> { + use std::task::Poll; + + let mut this = self.project(); + + if *this.done { + return Poll::Ready(None); + } + + loop { + let (input, payload) = + match Self::parse_next(this.buf, this.signing.is_some(), *this.has_trailer) { + Ok(res) => res, + Err(nom::Err::Incomplete(_)) => { + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + continue; + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(StreamingPayloadError::Stream(e)))) + } + None => { + return Poll::Ready(Some(Err(StreamingPayloadError::message( + "Unexpected EOF", + )))); + } + } + } + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + return Poll::Ready(Some(Err(e))) + } + }; + + match payload { + StreamingPayloadChunk::Chunk { data, header } => { + if let Some(signing) = this.signing.as_mut() { + let data_sha256sum = sha256sum(&data); + + let expected_signature = compute_streaming_payload_signature( + &signing.signing_hmac, + signing.datetime, + &signing.scope, + signing.previous_signature, + data_sha256sum, + )?; + + if header.signature.unwrap() != expected_signature { + return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature))); + } + + signing.previous_signature = header.signature.unwrap(); + } + + *this.buf = input.into(); + + // 0-sized chunk is the last + if data.is_empty() { + // if there was a trailer, it would have been returned by the parser + assert!(!*this.has_trailer); + *this.done = true; + return Poll::Ready(None); + } + + return Poll::Ready(Some(Ok(Frame::data(data)))); + } + StreamingPayloadChunk::Trailer(trailer) => { + trace!( + "In StreamingPayloadStream::poll_next: got trailer {:?}", + trailer + ); + + if let Some(signing) = this.signing.as_mut() { + let data = [ + trailer.header_name.as_ref(), + &b":"[..], + trailer.header_value.as_ref(), + &b"\n"[..], + ] + .concat(); + let trailer_sha256sum = sha256sum(&data); + + let expected_signature = compute_streaming_trailer_signature( + &signing.signing_hmac, + signing.datetime, + &signing.scope, + signing.previous_signature, + trailer_sha256sum, + )?; + + if trailer.signature.unwrap() != expected_signature { + return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature))); + } + } + + *this.buf = input.into(); + *this.done = true; + + let mut trailers_map = HeaderMap::new(); + trailers_map.insert(trailer.header_name, trailer.header_value); + + return Poll::Ready(Some(Ok(Frame::trailers(trailers_map)))); + } + } + } + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} + +#[cfg(test)] +mod tests { + use futures::prelude::*; + + use super::{SignParams, StreamingPayloadError, StreamingPayloadStream}; + + #[tokio::test] + async fn test_interrupted_signed_payload_stream() { + use chrono::{DateTime, Utc}; + + use garage_util::data::Hash; + + let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 + .unwrap() + .with_timezone(&Utc); + let secret_key = "test"; + let region = "test"; + let scope = crate::signature::compute_scope(&datetime, region, "s3"); + let signing_hmac = + crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); + + let data: &[&[u8]] = &[b"1"]; + let body = futures::stream::iter(data.iter().map(|block| Ok(block.to_vec().into()))); + + let seed_signature = Hash::default(); + + let mut stream = StreamingPayloadStream::new( + body, + Some(SignParams { + signing_hmac, + datetime, + scope, + previous_signature: seed_signature, + }), + false, + ); + + assert!(stream.try_next().await.is_err()); + match stream.try_next().await { + Err(StreamingPayloadError::Message(msg)) if msg == "Unexpected EOF" => {} + item => panic!( + "Unexpected result, expected early EOF error, got {:?}", + item + ), + } + } +} diff --git a/src/api/generic_server.rs b/src/api/generic_server.rs deleted file mode 100644 index 757b85ec..00000000 --- a/src/api/generic_server.rs +++ /dev/null @@ -1,225 +0,0 @@ -use std::net::SocketAddr; -use std::sync::Arc; - -use async_trait::async_trait; - -use futures::future::Future; - -use hyper::header::HeaderValue; -use hyper::server::conn::AddrStream; -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server}; -use hyper::{HeaderMap, StatusCode}; - -use opentelemetry::{ - global, - metrics::{Counter, ValueRecorder}, - trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, - Context, KeyValue, -}; - -use garage_util::error::Error as GarageError; -use garage_util::forwarded_headers; -use garage_util::metrics::{gen_trace_id, RecordDuration}; - -pub(crate) trait ApiEndpoint: Send + Sync + 'static { - fn name(&self) -> &'static str; - fn add_span_attributes(&self, span: SpanRef<'_>); -} - -pub trait ApiError: std::error::Error + Send + Sync + 'static { - fn http_status_code(&self) -> StatusCode; - fn add_http_headers(&self, header_map: &mut HeaderMap); - fn http_body(&self, garage_region: &str, path: &str) -> Body; -} - -#[async_trait] -pub(crate) trait ApiHandler: Send + Sync + 'static { - const API_NAME: &'static str; - const API_NAME_DISPLAY: &'static str; - - type Endpoint: ApiEndpoint; - type Error: ApiError; - - fn parse_endpoint(&self, r: &Request) -> Result; - async fn handle( - &self, - req: Request, - endpoint: Self::Endpoint, - ) -> Result, Self::Error>; -} - -pub(crate) struct ApiServer { - region: String, - api_handler: A, - - // Metrics - request_counter: Counter, - error_counter: Counter, - request_duration: ValueRecorder, -} - -impl ApiServer { - pub fn new(region: String, api_handler: A) -> Arc { - let meter = global::meter("garage/api"); - Arc::new(Self { - region, - api_handler, - request_counter: meter - .u64_counter(format!("api.{}.request_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - error_counter: meter - .u64_counter(format!("api.{}.error_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints that resulted in errors", - A::API_NAME_DISPLAY - )) - .init(), - request_duration: meter - .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) - .with_description(format!( - "Duration of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - }) - } - - pub async fn run_server( - self: Arc, - bind_addr: SocketAddr, - shutdown_signal: impl Future, - ) -> Result<(), GarageError> { - let service = make_service_fn(|conn: &AddrStream| { - let this = self.clone(); - - let client_addr = conn.remote_addr(); - async move { - Ok::<_, GarageError>(service_fn(move |req: Request| { - let this = this.clone(); - - this.handler(req, client_addr) - })) - } - }); - - let server = Server::bind(&bind_addr).serve(service); - - let graceful = server.with_graceful_shutdown(shutdown_signal); - info!( - "{} API server listening on http://{}", - A::API_NAME_DISPLAY, - bind_addr - ); - - graceful.await?; - Ok(()) - } - - async fn handler( - self: Arc, - req: Request, - addr: SocketAddr, - ) -> Result, GarageError> { - let uri = req.uri().clone(); - - if let Ok(forwarded_for_ip_addr) = - forwarded_headers::handle_forwarded_for_headers(req.headers()) - { - info!( - "{} (via {}) {} {}", - forwarded_for_ip_addr, - addr, - req.method(), - uri - ); - } else { - info!("{} {} {}", addr, req.method(), uri); - } - debug!("{:?}", req); - - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer - .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) - .with_trace_id(gen_trace_id()) - .with_attributes(vec![ - KeyValue::new("method", format!("{}", req.method())), - KeyValue::new("uri", req.uri().to_string()), - ]) - .start(&tracer); - - let res = self - .handler_stage2(req) - .with_context(Context::current_with_span(span)) - .await; - - match res { - Ok(x) => { - debug!("{} {:?}", x.status(), x.headers()); - Ok(x) - } - Err(e) => { - let body: Body = e.http_body(&self.region, uri.path()); - let mut http_error_builder = Response::builder().status(e.http_status_code()); - - if let Some(header_map) = http_error_builder.headers_mut() { - e.add_http_headers(header_map) - } - - let http_error = http_error_builder.body(body)?; - - if e.http_status_code().is_server_error() { - warn!("Response: error {}, {}", e.http_status_code(), e); - } else { - info!("Response: error {}, {}", e.http_status_code(), e); - } - Ok(http_error) - } - } - } - - async fn handler_stage2(&self, req: Request) -> Result, A::Error> { - let endpoint = self.api_handler.parse_endpoint(&req)?; - debug!("Endpoint: {}", endpoint.name()); - - let current_context = Context::current(); - let current_span = current_context.span(); - current_span.update_name::(format!( - "{} API {}", - A::API_NAME_DISPLAY, - endpoint.name() - )); - current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); - endpoint.add_span_attributes(current_span); - - let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; - - let res = self - .api_handler - .handle(req, endpoint) - .record_duration(&self.request_duration, &metrics_tags[..]) - .await; - - self.request_counter.add(1, &metrics_tags[..]); - - let status_code = match &res { - Ok(r) => r.status(), - Err(e) => e.http_status_code(), - }; - if status_code.is_client_error() || status_code.is_server_error() { - self.error_counter.add( - 1, - &[ - metrics_tags[0].clone(), - KeyValue::new("status_code", status_code.as_str().to_string()), - ], - ); - } - - res - } -} diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml new file mode 100644 index 00000000..28f74ea3 --- /dev/null +++ b/src/api/k2v/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "garage_api_k2v" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "K2V API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model = { workspace = true, features = [ "k2v" ] } +garage_table.workspace = true +garage_util = { workspace = true, features = [ "k2v" ] } +garage_api_common.workspace = true + +base64.workspace = true +thiserror.workspace = true +tracing.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +percent-encoding.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index bb85b2e7..8e10d9a6 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -1,35 +1,34 @@ -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; - -use futures::future::Future; -use hyper::{Body, Method, Request, Response}; +use hyper::{body::Incoming as IncomingBody, Method, Request, Response}; +use tokio::sync::watch; use opentelemetry::{trace::SpanRef, KeyValue}; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; -use crate::generic_server::*; -use crate::k2v::error::*; +use garage_api_common::cors::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_request; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::batch::*; +use crate::error::*; +use crate::index::*; +use crate::item::*; +use crate::router::Endpoint; -use crate::helpers::*; -use crate::k2v::batch::*; -use crate::k2v::index::*; -use crate::k2v::item::*; -use crate::k2v::router::Endpoint; -use crate::s3::cors::*; +pub use garage_api_common::signature::streaming::ReqBody; +pub type ResBody = BoxBody; pub struct K2VApiServer { garage: Arc, } -pub(crate) struct K2VApiEndpoint { +pub struct K2VApiEndpoint { bucket_name: String, endpoint: Endpoint, } @@ -37,17 +36,16 @@ pub(crate) struct K2VApiEndpoint { impl K2VApiServer { pub async fn run( garage: Arc, - bind_addr: SocketAddr, + bind_addr: UnixOrTCPSocketAddress, s3_region: String, - shutdown_signal: impl Future, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { ApiServer::new(s3_region, K2VApiServer { garage }) - .run_server(bind_addr, shutdown_signal) + .run_server(bind_addr, None, must_exit) .await } } -#[async_trait] impl ApiHandler for K2VApiServer { const API_NAME: &'static str = "k2v"; const API_NAME_DISPLAY: &'static str = "K2V"; @@ -55,7 +53,7 @@ impl ApiHandler for K2VApiServer { type Endpoint = K2VApiEndpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { + fn parse_endpoint(&self, req: &Request) -> Result { let (endpoint, bucket_name) = Endpoint::from_request(req)?; Ok(K2VApiEndpoint { @@ -66,42 +64,38 @@ impl ApiHandler for K2VApiServer { async fn handle( &self, - req: Request, + req: Request, endpoint: K2VApiEndpoint, - ) -> Result, Error> { + ) -> Result, Error> { let K2VApiEndpoint { bucket_name, endpoint, } = endpoint; let garage = self.garage.clone(); - // The OPTIONS method is procesed early, before we even check for an API key + // The OPTIONS method is processed early, before we even check for an API key if let Endpoint::Options = endpoint { - return Ok(handle_options_s3api(garage, &req, Some(bucket_name)) + let options_res = handle_options_api(garage, &req, Some(bucket_name)) .await - .ok_or_bad_request("Error handling OPTIONS")?); + .ok_or_bad_request("Error handling OPTIONS")?; + return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "k2v", - )?; + let verified_request = verify_request(&garage, req, "k2v").await?; + let req = verified_request.request; + let api_key = verified_request.access_key; let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) - .await?; + .await + .map_err(helper_error_as_internal)?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -119,40 +113,42 @@ impl ApiHandler for K2VApiServer { // are always preflighted, i.e. the browser should make // an OPTIONS call before to check it is allowed let matching_cors_rule = match *req.method() { - Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req) - .ok_or_internal_error("Error looking up CORS rule")?, + Method::GET | Method::HEAD | Method::POST => { + find_matching_cors_rule(&bucket_params, &req) + .ok_or_internal_error("Error looking up CORS rule")? + .cloned() + } _ => None, }; + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + let resp = match endpoint { Endpoint::DeleteItem { partition_key, sort_key, - } => handle_delete_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_delete_item(ctx, req, &partition_key, &sort_key).await, Endpoint::InsertItem { partition_key, sort_key, - } => handle_insert_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_insert_item(ctx, req, &partition_key, &sort_key).await, Endpoint::ReadItem { partition_key, sort_key, - } => handle_read_item(garage, &req, bucket_id, &partition_key, &sort_key).await, + } => handle_read_item(ctx, &req, &partition_key, &sort_key).await, Endpoint::PollItem { partition_key, sort_key, causality_token, timeout, } => { - handle_poll_item( - garage, - &req, - bucket_id, - partition_key, - sort_key, - causality_token, - timeout, - ) - .await + handle_poll_item(ctx, &req, partition_key, sort_key, causality_token, timeout).await } Endpoint::ReadIndex { prefix, @@ -160,12 +156,12 @@ impl ApiHandler for K2VApiServer { end, limit, reverse, - } => handle_read_index(garage, bucket_id, prefix, start, end, limit, reverse).await, - Endpoint::InsertBatch {} => handle_insert_batch(garage, bucket_id, req).await, - Endpoint::ReadBatch {} => handle_read_batch(garage, bucket_id, req).await, - Endpoint::DeleteBatch {} => handle_delete_batch(garage, bucket_id, req).await, + } => handle_read_index(ctx, prefix, start, end, limit, reverse).await, + Endpoint::InsertBatch {} => handle_insert_batch(ctx, req).await, + Endpoint::ReadBatch {} => handle_read_batch(ctx, req).await, + Endpoint::DeleteBatch {} => handle_delete_batch(ctx, req).await, Endpoint::PollRange { partition_key } => { - handle_poll_range(garage, bucket_id, &partition_key, req).await + handle_poll_range(ctx, &partition_key, req).await } Endpoint::Options => unreachable!(), }; @@ -174,12 +170,18 @@ impl ApiHandler for K2VApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } Ok(resp_ok) } + + fn key_id_from_request(&self, req: &Request) -> Option { + garage_api_common::signature::payload::Authorization::parse_header(req.headers()) + .map(|auth| auth.key_id) + .ok() + } } impl ApiEndpoint for K2VApiEndpoint { diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 294380ea..7a03d836 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -1,31 +1,30 @@ -use std::sync::Arc; - use base64::prelude::*; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use garage_util::data::*; - use garage_table::{EnumerationOrder, TableSchema}; -use garage_model::garage::Garage; -use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::helpers::*; -use crate::k2v::error::*; -use crate::k2v::range::read_range; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::item::parse_causality_token; +use crate::range::read_range; pub async fn handle_insert_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let items = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let items = req.into_body().json::>().await?; let mut items2 = vec![]; for it in items { - let ct = it.ct.map(|s| CausalContext::parse_helper(&s)).transpose()?; + let ct = it.ct.map(|s| parse_causality_token(&s)).transpose()?; let v = match it.v { Some(vs) => DvvsValue::Value( BASE64_STANDARD @@ -37,24 +36,23 @@ pub async fn handle_insert_batch( items2.push((it.pk, it.sk, ct, v)); } - garage.k2v.rpc.insert_batch(bucket_id, items2).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items2).await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_read_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let queries = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let queries = req.into_body().json::>().await?; let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_read_batch_query(&garage, bucket_id, q)), + .map(|q| handle_read_batch_query(&ctx, q)), ) .await; @@ -67,12 +65,15 @@ pub async fn handle_read_batch( } async fn handle_read_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: ReadBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -137,16 +138,15 @@ async fn handle_read_batch_query( } pub async fn handle_delete_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let queries = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let queries = req.into_body().json::>().await?; let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_delete_batch_query(&garage, bucket_id, q)), + .map(|q| handle_delete_batch_query(&ctx, q)), ) .await; @@ -159,12 +159,15 @@ pub async fn handle_delete_batch( } async fn handle_delete_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: DeleteBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -194,7 +197,7 @@ async fn handle_delete_batch_query( .k2v .rpc .insert( - bucket_id, + *bucket_id, i.partition.partition_key, i.sort_key, Some(cc), @@ -234,7 +237,7 @@ async fn handle_delete_batch_query( .collect::>(); let n = items.len(); - garage.k2v.rpc.insert_batch(bucket_id, items).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items).await?; n }; @@ -250,14 +253,16 @@ async fn handle_delete_batch_query( } pub(crate) async fn handle_poll_range( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, partition_key: &str, - req: Request, -) -> Result, Error> { + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; use garage_model::k2v::sub::PollRange; - let query = parse_json_body::(req).await?; + let query = req.into_body().json::().await?; let timeout_msec = query.timeout.unwrap_or(300).clamp(1, 600) * 1000; @@ -277,7 +282,8 @@ pub(crate) async fn handle_poll_range( query.seen_marker, timeout_msec, ) - .await?; + .await + .map_err(pass_helper_error)?; if let Some((items, seen_marker)) = resp { let resp = PollRangeResponse { @@ -292,7 +298,7 @@ pub(crate) async fn handle_poll_range( } else { Ok(Response::builder() .status(StatusCode::NOT_MODIFIED) - .body(Body::empty())?) + .body(empty_body())?) } } diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 4eb017ab..f1937fe5 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -1,70 +1,54 @@ -use err_derive::Error; use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; -use garage_model::helper::error::Error as HelperError; - -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::CustomApiErrorBody; -use crate::signature::error::Error as SignatureError; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; +pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error}; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), + /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// Some base64 encoded data was badly encoded - #[error(display = "Invalid base64: {}", _0)] - InvalidBase64(#[error(source)] base64::DecodeError), + #[error("Invalid base64: {0}")] + InvalidBase64(#[from] base64::DecodeError), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// Invalid causality token + #[error("Invalid causality token")] + InvalidCausalityToken, /// The client asked for an invalid return format (invalid Accept header) - #[error(display = "Not acceptable: {}", _0)] + #[error("Not acceptable: {0}")] NotAcceptable(String), /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} - -impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - e => Self::Common(CommonError::BadRequest(format!("{}", e))), - } - } -} +commonErrorDerivative!(Error); impl From for Error { fn from(err: SignatureError) -> Self { @@ -74,7 +58,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), - SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } @@ -90,8 +74,9 @@ impl Error { Error::NotAcceptable(_) => "NotAcceptable", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", Error::InvalidBase64(_) => "InvalidBase64", - Error::InvalidHeader(_) => "InvalidHeaderValue", Error::InvalidUtf8Str(_) => "InvalidUtf8String", + Error::InvalidCausalityToken => "CausalityToken", + Error::InvalidDigest(_) => "InvalidDigest", } } } @@ -105,8 +90,9 @@ impl ApiError for Error { Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, Error::AuthorizationHeaderMalformed(_) | Error::InvalidBase64(_) - | Error::InvalidHeader(_) - | Error::InvalidUtf8Str(_) => StatusCode::BAD_REQUEST, + | Error::InvalidUtf8Str(_) + | Error::InvalidDigest(_) + | Error::InvalidCausalityToken => StatusCode::BAD_REQUEST, } } @@ -115,14 +101,14 @@ impl ApiError for Error { header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap()); } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = CustomApiErrorBody { code: self.code().to_string(), message: format!("{}", self), path: path.to_string(), region: garage_region.to_string(), }; - Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + let error_str = serde_json::to_string_pretty(&error).unwrap_or_else(|_| { r#" { "code": "InternalError", @@ -130,6 +116,7 @@ impl ApiError for Error { } "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index 6c1d4a91..fbfaad98 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -1,32 +1,35 @@ -use std::sync::Arc; - -use hyper::{Body, Response}; +use hyper::Response; use serde::Serialize; -use garage_util::data::*; - -use garage_rpc::ring::Ring; use garage_table::util::*; -use garage_model::garage::Garage; use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; -use crate::helpers::*; -use crate::k2v::error::*; -use crate::k2v::range::read_range; +use garage_api_common::helpers::*; + +use crate::api_server::ResBody; +use crate::error::*; +use crate::range::read_range; pub async fn handle_read_index( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, prefix: Option, start: Option, end: Option, limit: Option, reverse: Option, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let reverse = reverse.unwrap_or(false); - let ring: Arc = garage.system.ring.borrow().clone(); + let node_id_vec = garage + .system + .cluster_layout() + .all_nongateway_nodes() + .to_vec(); let (partition_keys, more, next_start) = read_range( &garage.k2v.counter_table.table, @@ -35,7 +38,7 @@ pub async fn handle_read_index( &start, &end, limit, - Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())), + Some((DeletedFilter::NotDeleted, node_id_vec)), EnumerationOrder::from_reverse(reverse), ) .await?; @@ -54,7 +57,7 @@ pub async fn handle_read_index( partition_keys: partition_keys .into_iter() .map(|part| { - let vals = part.filtered_values(&ring); + let vals = part.filtered_values(&garage.system.cluster_layout()); ReadIndexResponseEntry { pk: part.sk, entries: *vals.get(&s_entries).unwrap_or(&0), @@ -68,7 +71,7 @@ pub async fn handle_read_index( next_start, }; - Ok(json_ok_response(&resp)?) + json_ok_response::(&resp) } #[derive(Serialize)] diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index e13a0f30..0fb945d2 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -1,17 +1,15 @@ -use std::sync::Arc; - use base64::prelude::*; use http::header; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; -use garage_util::data::*; - -use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::k2v::error::*; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; @@ -21,8 +19,12 @@ pub enum ReturnFormat { Either, } +pub(crate) fn parse_causality_token(s: &str) -> Result { + CausalContext::parse(s).ok_or(Error::InvalidCausalityToken) +} + impl ReturnFormat { - pub fn from(req: &Request) -> Result { + pub fn from(req: &Request) -> Result { let accept = match req.headers().get(header::ACCEPT) { Some(a) => a.to_str()?, None => return Ok(Self::Json), @@ -40,7 +42,7 @@ impl ReturnFormat { } } - pub fn make_response(&self, item: &K2VItem) -> Result, Error> { + pub fn make_response(&self, item: &K2VItem) -> Result, Error> { let vals = item.values(); if vals.is_empty() { @@ -52,7 +54,7 @@ impl ReturnFormat { Self::Binary if vals.len() > 1 => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .status(StatusCode::CONFLICT) - .body(Body::empty())?), + .body(empty_body())?), Self::Binary => { assert!(vals.len() == 1); Self::make_binary_response(ct, vals[0]) @@ -62,22 +64,22 @@ impl ReturnFormat { } } - fn make_binary_response(ct: String, v: &DvvsValue) -> Result, Error> { + fn make_binary_response(ct: String, v: &DvvsValue) -> Result, Error> { match v { DvvsValue::Deleted => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/octet-stream") .status(StatusCode::NO_CONTENT) - .body(Body::empty())?), + .body(empty_body())?), DvvsValue::Value(v) => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/octet-stream") .status(StatusCode::OK) - .body(Body::from(v.to_vec()))?), + .body(bytes_body(v.to_vec().into()))?), } } - fn make_json_response(ct: String, v: &[&DvvsValue]) -> Result, Error> { + fn make_json_response(ct: String, v: &[&DvvsValue]) -> Result, Error> { let items = v .iter() .map(|v| match v { @@ -91,19 +93,22 @@ impl ReturnFormat { .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/json") .status(StatusCode::OK) - .body(Body::from(json_body))?) + .body(string_body(json_body))?) } } /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_read_item( - garage: Arc, - req: &Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, partition_key: &str, sort_key: &String, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let format = ReturnFormat::from(req)?; let item = garage @@ -111,7 +116,7 @@ pub async fn handle_read_item( .item_table .get( &K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: partition_key.to_string(), }, sort_key, @@ -123,28 +128,31 @@ pub async fn handle_read_item( } pub async fn handle_insert_item( - garage: Arc, - req: Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: Request, partition_key: &str, sort_key: &str, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) .map(|s| s.to_str()) .transpose()? - .map(CausalContext::parse_helper) + .map(parse_causality_token) .transpose()?; - let body = hyper::body::to_bytes(req.into_body()).await?; + let body = req.into_body().collect().await?; + let value = DvvsValue::Value(body.to_vec()); garage .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -154,22 +162,24 @@ pub async fn handle_insert_item( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_delete_item( - garage: Arc, - req: Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: Request, partition_key: &str, sort_key: &str, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) .map(|s| s.to_str()) .transpose()? - .map(CausalContext::parse_helper) + .map(parse_causality_token) .transpose()?; let value = DvvsValue::Deleted; @@ -178,7 +188,7 @@ pub async fn handle_delete_item( .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -188,20 +198,22 @@ pub async fn handle_delete_item( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_poll_item( - garage: Arc, - req: &Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, partition_key: String, sort_key: String, causality_token: String, timeout_secs: Option, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let format = ReturnFormat::from(req)?; let causal_context = @@ -213,7 +225,7 @@ pub async fn handle_poll_item( .k2v .rpc .poll_item( - bucket_id, + *bucket_id, partition_key, sort_key, causal_context, @@ -226,6 +238,6 @@ pub async fn handle_poll_item( } else { Ok(Response::builder() .status(StatusCode::NOT_MODIFIED) - .body(Body::empty())?) + .body(empty_body())?) } } diff --git a/src/api/k2v/mod.rs b/src/api/k2v/lib.rs similarity index 70% rename from src/api/k2v/mod.rs rename to src/api/k2v/lib.rs index b6a8c5cf..334ae46b 100644 --- a/src/api/k2v/mod.rs +++ b/src/api/k2v/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; mod error; mod router; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index bb9d3be5..eb4738db 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::helpers::key_after_prefix; -use crate::k2v::error::*; +use garage_api_common::helpers::key_after_prefix; + +use crate::error::*; /// Read range in a Garage table. /// Returns (entries, more?, nextStart) diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index 1cc58be5..a04b0f81 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -1,11 +1,11 @@ -use crate::k2v::error::*; +use crate::error::*; use std::borrow::Cow; use hyper::{Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; router_match! {@func diff --git a/src/api/lib.rs b/src/api/lib.rs deleted file mode 100644 index 370dfd7a..00000000 --- a/src/api/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Crate for serving a S3 compatible API -#[macro_use] -extern crate tracing; - -pub mod common_error; - -mod encoding; -pub mod generic_server; -pub mod helpers; -mod router_macros; -/// This mode is public only to help testing. Don't expect stability here -pub mod signature; - -pub mod admin; -#[cfg(feature = "k2v")] -pub mod k2v; -pub mod s3; diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml new file mode 100644 index 00000000..88630866 --- /dev/null +++ b/src/api/s3/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "garage_api_s3" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +aes-gcm.workspace = true +async-compression.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +thiserror.workspace = true +hex.workspace = true +tracing.workspace = true +md-5.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 3f995d34..acb0cf56 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -1,44 +1,44 @@ -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; - -use futures::future::Future; use hyper::header; -use hyper::{Body, Request, Response}; +use hyper::{body::Incoming as IncomingBody, Request, Response}; +use tokio::sync::watch; use opentelemetry::{trace::SpanRef, KeyValue}; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::generic_server::*; -use crate::s3::error::*; +use garage_api_common::cors::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_request; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::bucket::*; +use crate::copy::*; +use crate::cors::*; +use crate::delete::*; +use crate::error::*; +use crate::get::*; +use crate::lifecycle::*; +use crate::list::*; +use crate::multipart::*; +use crate::post_object::handle_post_object; +use crate::put::*; +use crate::router::Endpoint; +use crate::website::*; -use crate::helpers::*; -use crate::s3::bucket::*; -use crate::s3::copy::*; -use crate::s3::cors::*; -use crate::s3::delete::*; -use crate::s3::get::*; -use crate::s3::lifecycle::*; -use crate::s3::list::*; -use crate::s3::multipart::*; -use crate::s3::post_object::handle_post_object; -use crate::s3::put::*; -use crate::s3::router::Endpoint; -use crate::s3::website::*; +pub use garage_api_common::signature::streaming::ReqBody; +pub type ResBody = BoxBody; pub struct S3ApiServer { garage: Arc, } -pub(crate) struct S3ApiEndpoint { +pub struct S3ApiEndpoint { bucket_name: Option, endpoint: Endpoint, } @@ -46,21 +46,21 @@ pub(crate) struct S3ApiEndpoint { impl S3ApiServer { pub async fn run( garage: Arc, - addr: SocketAddr, + addr: UnixOrTCPSocketAddress, s3_region: String, - shutdown_signal: impl Future, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { ApiServer::new(s3_region, S3ApiServer { garage }) - .run_server(addr, shutdown_signal) + .run_server(addr, None, must_exit) .await } async fn handle_request_without_bucket( &self, - _req: Request, + _req: Request, api_key: Key, endpoint: Endpoint, - ) -> Result, Error> { + ) -> Result, Error> { match endpoint { Endpoint::ListBuckets => handle_list_buckets(&self.garage, &api_key).await, endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), @@ -68,7 +68,6 @@ impl S3ApiServer { } } -#[async_trait] impl ApiHandler for S3ApiServer { const API_NAME: &'static str = "s3"; const API_NAME_DISPLAY: &'static str = "S3"; @@ -76,7 +75,7 @@ impl ApiHandler for S3ApiServer { type Endpoint = S3ApiEndpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { + fn parse_endpoint(&self, req: &Request) -> Result { let authority = req .headers() .get(header::HOST) @@ -104,9 +103,9 @@ impl ApiHandler for S3ApiServer { async fn handle( &self, - req: Request, + req: Request, endpoint: S3ApiEndpoint, - ) -> Result, Error> { + ) -> Result, Error> { let S3ApiEndpoint { bucket_name, endpoint, @@ -118,20 +117,13 @@ impl ApiHandler for S3ApiServer { return handle_post_object(garage, req, bucket_name.unwrap()).await; } if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, bucket_name).await; + let options_res = handle_options_api(garage, &req, bucket_name).await?; + return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "s3", - )?; + let verified_request = verify_request(&garage, req, "s3").await?; + let req = verified_request.request; + let api_key = verified_request.access_key; let bucket_name = match bucket_name { None => { @@ -144,17 +136,19 @@ impl ApiHandler for S3ApiServer { // Special code path for CreateBucket API endpoint if let Endpoint::CreateBucket {} = endpoint { - return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await; + return handle_create_bucket(&garage, req, &api_key.key_id, bucket_name).await; } let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -167,83 +161,72 @@ impl ApiHandler for S3ApiServer { return Err(Error::forbidden("Operation is not allowed for this key.")); } - let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?; + let matching_cors_rule = find_matching_cors_rule(&bucket_params, &req)?.cloned(); + + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; let resp = match endpoint { Endpoint::HeadObject { key, part_number, .. - } => handle_head(garage, &req, bucket_id, &key, part_number).await, + } => handle_head(ctx, &req.map(|_| ()), &key, part_number).await, Endpoint::GetObject { - key, part_number, .. - } => handle_get(garage, &req, bucket_id, &key, part_number).await, + key, + part_number, + response_cache_control, + response_content_disposition, + response_content_encoding, + response_content_language, + response_content_type, + response_expires, + .. + } => { + let overrides = GetObjectOverrides { + response_cache_control, + response_content_disposition, + response_content_encoding, + response_content_language, + response_content_type, + response_expires, + }; + handle_get(ctx, &req.map(|_| ()), &key, part_number, overrides).await + } Endpoint::UploadPart { key, part_number, upload_id, - } => { - handle_put_part( - garage, - req, - bucket_id, - &key, - part_number, - &upload_id, - content_sha256, - ) - .await - } - Endpoint::CopyObject { key } => { - handle_copy(garage, &api_key, &req, bucket_id, &key).await - } + } => handle_put_part(ctx, req, &key, part_number, &upload_id).await, + Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await, Endpoint::UploadPartCopy { key, part_number, upload_id, - } => { - handle_upload_part_copy( - garage, - &api_key, - &req, - bucket_id, - &key, - part_number, - &upload_id, - ) - .await - } - Endpoint::PutObject { key } => { - handle_put(garage, req, &bucket, &key, content_sha256).await - } + } => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await, + Endpoint::PutObject { key } => handle_put(ctx, req, &key).await, Endpoint::AbortMultipartUpload { key, upload_id } => { - handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await + handle_abort_multipart_upload(ctx, &key, &upload_id).await } - Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await, + Endpoint::DeleteObject { key, .. } => handle_delete(ctx, &key).await, Endpoint::CreateMultipartUpload { key } => { - handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await + handle_create_multipart_upload(ctx, &req, &key).await } Endpoint::CompleteMultipartUpload { key, upload_id } => { - handle_complete_multipart_upload( - garage, - req, - &bucket_name, - &bucket, - &key, - &upload_id, - content_sha256, - ) - .await + handle_complete_multipart_upload(ctx, req, &key, &upload_id).await } Endpoint::CreateBucket {} => unreachable!(), Endpoint::HeadBucket {} => { - let empty_body: Body = Body::from(vec![]); - let response = Response::builder().body(empty_body).unwrap(); + let response = Response::builder().body(empty_body()).unwrap(); Ok(response) } - Endpoint::DeleteBucket {} => { - handle_delete_bucket(&garage, bucket_id, bucket_name, api_key).await - } - Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage), + Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await, + Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx), Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(), + Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx), Endpoint::ListObjects { delimiter, encoding_type, @@ -251,24 +234,21 @@ impl ApiHandler for S3ApiServer { max_keys, prefix, } => { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_keys.unwrap_or(1000).clamp(1, 1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - is_v2: false, - marker, - continuation_token: None, - start_after: None, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + is_v2: false, + marker, + continuation_token: None, + start_after: None, + }; + handle_list(ctx, &query).await } Endpoint::ListObjectsV2 { delimiter, @@ -281,24 +261,21 @@ impl ApiHandler for S3ApiServer { .. } => { if list_type == "2" { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_keys.unwrap_or(1000).clamp(1, 1000), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - prefix: prefix.unwrap_or_default(), - }, - is_v2: true, - marker: None, - continuation_token, - start_after, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), + prefix: prefix.unwrap_or_default(), }, - ) - .await + is_v2: true, + marker: None, + continuation_token, + start_after, + }; + handle_list(ctx, &query).await } else { Err(Error::bad_request(format!( "Invalid endpoint: list-type={}", @@ -314,22 +291,19 @@ impl ApiHandler for S3ApiServer { prefix, upload_id_marker, } => { - handle_list_multipart_upload( - garage, - &ListMultipartUploadsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_uploads.unwrap_or(1000).clamp(1, 1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - key_marker, - upload_id_marker, + let query = ListMultipartUploadsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_uploads.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + key_marker, + upload_id_marker, + }; + handle_list_multipart_upload(ctx, &query).await } Endpoint::ListParts { key, @@ -337,39 +311,25 @@ impl ApiHandler for S3ApiServer { part_number_marker, upload_id, } => { - handle_list_parts( - garage, - &ListPartsQuery { - bucket_name, - bucket_id, - key, - upload_id, - part_number_marker: part_number_marker.map(|p| p.clamp(1, 10000)), - max_parts: max_parts.unwrap_or(1000).clamp(1, 1000), - }, - ) - .await - } - Endpoint::DeleteObjects {} => { - handle_delete_objects(garage, bucket_id, req, content_sha256).await - } - Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await, - Endpoint::PutBucketWebsite {} => { - handle_put_website(garage, bucket.clone(), req, content_sha256).await - } - Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket.clone()).await, - Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await, - Endpoint::PutBucketCors {} => { - handle_put_cors(garage, bucket.clone(), req, content_sha256).await - } - Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket.clone()).await, - Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(&bucket).await, - Endpoint::PutBucketLifecycleConfiguration {} => { - handle_put_lifecycle(garage, bucket.clone(), req, content_sha256).await - } - Endpoint::DeleteBucketLifecycle {} => { - handle_delete_lifecycle(garage, bucket.clone()).await + let query = ListPartsQuery { + bucket_name: ctx.bucket_name.clone(), + key, + upload_id, + part_number_marker: part_number_marker.map(|p| p.min(10000)), + max_parts: max_parts.unwrap_or(1000).clamp(1, 1000), + }; + handle_list_parts(ctx, req, &query).await } + Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req).await, + Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await, + Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req).await, + Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await, + Endpoint::GetBucketCors {} => handle_get_cors(ctx).await, + Endpoint::PutBucketCors {} => handle_put_cors(ctx, req).await, + Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await, + Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await, + Endpoint::PutBucketLifecycleConfiguration {} => handle_put_lifecycle(ctx, req).await, + Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await, endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), }; @@ -377,12 +337,18 @@ impl ApiHandler for S3ApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } Ok(resp_ok) } + + fn key_id_from_request(&self, req: &Request) -> Option { + garage_api_common::signature::payload::Authorization::parse_header(req.headers()) + .map(|auth| auth.key_id) + .ok() + } } impl ApiEndpoint for S3ApiEndpoint { diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 733981e1..55caa6c8 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -1,24 +1,25 @@ use std::collections::HashMap; -use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use garage_model::bucket_alias_table::*; use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; -use garage_model::key_table::Key; +use garage_model::key_table::{Key, KeyParams}; use garage_model::permission::BucketKeyPerm; use garage_table::util::*; use garage_util::crdt::*; -use garage_util::data::*; use garage_util::time::*; -use crate::common_error::CommonError; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; -pub fn handle_get_bucket_location(garage: Arc) -> Result, Error> { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; + +pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { garage, .. } = ctx; let loc = s3_xml::LocationConstraint { xmlns: (), region: garage.config.s3_api.s3_region.to_string(), @@ -27,10 +28,10 @@ pub fn handle_get_bucket_location(garage: Arc) -> Result, Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } -pub fn handle_get_bucket_versioning() -> Result, Error> { +pub fn handle_get_bucket_versioning() -> Result, Error> { let versioning = s3_xml::VersioningConfiguration { xmlns: (), status: None, @@ -40,10 +41,62 @@ pub fn handle_get_bucket_versioning() -> Result, Error> { Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } -pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result, Error> { +pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + bucket_id, api_key, .. + } = ctx; + let key_p = api_key.params().ok_or_internal_error( + "Key should not be in deleted state at this point (in handle_get_bucket_acl)", + )?; + + let mut grants: Vec = vec![]; + let kp = api_key.bucket_permissions(&bucket_id); + + if kp.allow_owner { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("FULL_CONTROL".to_string()), + }); + } else { + if kp.allow_read { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ".to_string()), + }); + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ_ACP".to_string()), + }); + } + if kp.allow_write { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("WRITE".to_string()), + }); + } + } + + let access_control_policy = s3_xml::AccessControlPolicy { + xmlns: (), + owner: None, + acl: s3_xml::AccessControlList { entries: grants }, + }; + + let xml = s3_xml::to_xml_with_header(&access_control_policy)?; + trace!("xml: {}", xml); + + Ok(Response::builder() + .header("Content-Type", "application/xml") + .body(string_body(xml))?) +} + +pub async fn handle_list_buckets( + garage: &Garage, + api_key: &Key, +) -> Result, Error> { let key_p = api_key.params().ok_or_internal_error( "Key should not be in deleted state at this point (in handle_list_buckets)", )?; @@ -109,21 +162,16 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result, - content_sha256: Option, - api_key: Key, + req: Request, + api_key_id: &String, bucket_name: String, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } +) -> Result, Error> { + let body = req.into_body().collect().await?; let cmd = parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?; @@ -138,16 +186,18 @@ pub async fn handle_create_bucket( } } - let key_params = api_key - .params() - .ok_or_internal_error("Key should not be deleted at this point")?; + let helper = garage.locked_helper().await; + + // refetch API key after taking lock to ensure up-to-date data + let api_key = helper.key().get_existing_key(api_key_id).await?; + let key_params = api_key.params().unwrap(); let existing_bucket = if let Some(Some(bucket_id)) = key_params.local_aliases.get(&bucket_name) { Some(*bucket_id) } else { - garage - .bucket_helper() + helper + .bucket() .resolve_global_bucket_name(&bucket_name) .await? }; @@ -171,7 +221,7 @@ pub async fn handle_create_bucket( } // Create the bucket! - if !is_valid_bucket_name(&bucket_name) { + if !is_valid_bucket_name(&bucket_name, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", bucket_name, INVALID_BUCKET_NAME_MESSAGE @@ -181,40 +231,35 @@ pub async fn handle_create_bucket( let bucket = Bucket::new(); garage.bucket_table.insert(&bucket).await?; - garage - .bucket_helper() + helper .set_bucket_key_permissions(bucket.id, &api_key.key_id, BucketKeyPerm::ALL_PERMISSIONS) .await?; - garage - .bucket_helper() + helper .set_local_bucket_alias(bucket.id, &api_key.key_id, &bucket_name) .await?; } Ok(Response::builder() .header("Location", format!("/{}", bucket_name)) - .body(Body::empty()) + .body(empty_body()) .unwrap()) } -pub async fn handle_delete_bucket( - garage: &Garage, - bucket_id: Uuid, - bucket_name: String, - api_key: Key, -) -> Result, Error> { - let key_params = api_key - .params() - .ok_or_internal_error("Key should not be deleted at this point")?; +pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params: bucket_state, + api_key, + .. + } = &ctx; + let helper = garage.locked_helper().await; - let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_))); + let key_params = api_key.params().unwrap(); - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - let bucket_state = bucket.state.as_option().unwrap(); + let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_))); // If the bucket has no other aliases, this is a true deletion. // Otherwise, it is just an alias removal. @@ -224,65 +269,63 @@ pub async fn handle_delete_bucket( .items() .iter() .filter(|(_, _, active)| *active) - .any(|(n, _, _)| is_local_alias || (*n != bucket_name)); + .any(|(n, _, _)| is_local_alias || (*n != *bucket_name)); let has_other_local_aliases = bucket_state .local_aliases .items() .iter() .filter(|(_, _, active)| *active) - .any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id); + .any(|((k, n), _, _)| !is_local_alias || *n != *bucket_name || *k != api_key.key_id); if !has_other_global_aliases && !has_other_local_aliases { // Delete bucket // Check bucket is empty - if !garage.bucket_helper().is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(*bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } // --- done checking, now commit --- // 1. delete bucket alias if is_local_alias { - garage - .bucket_helper() - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + helper + .purge_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &bucket_name) + helper + .purge_global_bucket_alias(*bucket_id, bucket_name) .await?; } // 2. delete authorization from keys that had access - for (key_id, _) in bucket.authorized_keys() { - garage - .bucket_helper() - .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) + for (key_id, _) in bucket_state.authorized_keys.items() { + helper + .set_bucket_key_permissions(*bucket_id, key_id, BucketKeyPerm::NO_PERMISSIONS) .await?; } + let bucket = Bucket { + id: *bucket_id, + state: Deletable::delete(), + }; // 3. delete bucket - bucket.state = Deletable::delete(); garage.bucket_table.insert(&bucket).await?; } else if is_local_alias { // Just unalias - garage - .bucket_helper() - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + helper + .unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { // Just unalias (but from global namespace) - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &bucket_name) + helper + .unset_global_bucket_alias(*bucket_id, bucket_name) .await?; } Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option> { @@ -317,6 +360,15 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option> { Some(ret) } +fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee { + s3_xml::Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(s3_xml::Value(key_params.name.get().to_string())), + id: Some(s3_xml::Value(api_key.key_id.to_string())), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 68b4f0c9..47a63c82 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -1,70 +1,220 @@ use std::pin::Pin; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use futures::{stream, stream::Stream, StreamExt}; -use md5::{Digest as Md5Digest, Md5}; +use futures::{stream, stream::Stream, StreamExt, TryStreamExt}; use bytes::Bytes; -use hyper::{Body, Request, Response}; +use http::header::HeaderName; +use hyper::{Request, Response}; use serde::Serialize; -use garage_rpc::netapp::bytes_buf::BytesBuf; +use garage_net::bytes_buf::BytesBuf; +use garage_net::stream::read_stream_to_end; use garage_rpc::rpc_helper::OrderTag; use garage_table::*; use garage_util::data::*; +use garage_util::error::Error as GarageError; use garage_util::time::*; -use garage_model::garage::Garage; -use garage_model::key_table::Key; use garage_model::s3::block_ref_table::*; use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::parse_bucket_key; -use crate::s3::error::*; -use crate::s3::multipart; -use crate::s3::put::get_headers; -use crate::s3::xml::{self as s3_xml, xmlns_tag}; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders}; +use crate::multipart; +use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult}; +use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; +use crate::xml::{self as s3_xml, xmlns_tag}; + +pub const X_AMZ_COPY_SOURCE_IF_MATCH: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-match"); +pub const X_AMZ_COPY_SOURCE_IF_NONE_MATCH: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-none-match"); +pub const X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-modified-since"); +pub const X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-unmodified-since"); + +// -------- CopyObject --------- pub async fn handle_copy( - garage: Arc, - api_key: &Key, - req: &Request, - dest_bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, dest_key: &str, -) -> Result, Error> { - let copy_precondition = CopyPreconditionHeaders::parse(req)?; +) -> Result, Error> { + let copy_precondition = PreconditionHeaders::parse_copy_source(req)?; - let source_object = get_copy_source(&garage, api_key, req).await?; + let checksum_algorithm = request_checksum_algorithm(req.headers())?; + + let source_object = get_copy_source(&ctx, req).await?; let (source_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; // Check precondition, e.g. x-amz-copy-source-if-match - copy_precondition.check(source_version, &source_version_meta.etag)?; + copy_precondition.check_copy_source(source_version, &source_version_meta.etag)?; + + // Determine encryption parameters + let (source_encryption, source_object_meta_inner) = + EncryptionParams::check_decrypt_for_copy_source( + &ctx.garage, + req.headers(), + &source_version_meta.encryption, + )?; + let dest_encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?; + + // Extract source checksum info before source_object_meta_inner is consumed + let source_checksum = source_object_meta_inner.checksum; + let source_checksum_algorithm = source_checksum.map(|x| x.algorithm()); + + // If source object has a checksum, the destination object must as well. + // The x-amz-checksum-algorithm header allows to change that algorithm, + // but if it is absent, we must use the same as before + let checksum_algorithm = checksum_algorithm.or(source_checksum_algorithm); + + // Determine metadata of destination object + let was_multipart = source_version_meta.etag.contains('-'); + let dest_object_meta = ObjectVersionMetaInner { + headers: match req.headers().get("x-amz-metadata-directive") { + Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => { + extract_metadata_headers(req.headers())? + } + _ => { + // The x-amz-website-redirect-location header is not copied, instead + // it is replaced by the value from the request (or removed if no + // value was specified) + let is_redirect = + |(key, _): &(String, String)| key == X_AMZ_WEBSITE_REDIRECT_LOCATION.as_str(); + let mut headers: Vec<_> = source_object_meta_inner.headers.clone(); + headers.retain(|h| !is_redirect(h)); + let new_headers = extract_metadata_headers(req.headers())?; + headers.extend(new_headers.into_iter().filter(is_redirect)); + headers + } + }, + checksum: source_checksum, + }; + + // Do actual object copying + // + // In any of the following scenarios, we need to read the whole object + // data and re-write it again: + // + // - the data needs to be decrypted or encrypted + // - the requested checksum algorithm requires us to recompute a checksum + // - the original object was a multipart upload and a checksum algorithm + // is defined (AWS specifies that in this case, we must recompute the + // checksum from scratch as if this was a single big object and not + // a multipart object, as the checksums are not computed in the same way) + // + // In other cases, we can just copy the metadata and reference the same blocks. + // + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + + let must_recopy = !EncryptionParams::is_same(&source_encryption, &dest_encryption) + || source_checksum_algorithm != checksum_algorithm + || (was_multipart && checksum_algorithm.is_some()); + + let res = if !must_recopy { + // In most cases, we can just copy the metadata and link blocks of the + // old object from the new object. + handle_copy_metaonly( + ctx, + dest_key, + dest_object_meta, + dest_encryption, + source_version, + source_version_data, + source_version_meta, + ) + .await? + } else { + let expected_checksum = ExpectedChecksums { + md5: None, + sha256: None, + extra: source_checksum, + }; + let checksum_mode = if was_multipart || source_checksum_algorithm != checksum_algorithm { + ChecksumMode::Calculate(checksum_algorithm) + } else { + ChecksumMode::Verify(&expected_checksum) + }; + // If source and dest encryption use different keys, + // we must decrypt content and re-encrypt, so rewrite all data blocks. + handle_copy_reencrypt( + ctx, + dest_key, + dest_object_meta, + dest_encryption, + source_version, + source_version_data, + source_encryption, + checksum_mode, + ) + .await? + }; + + let last_modified = msec_to_rfc3339(res.version_timestamp); + let result = CopyObjectResult { + last_modified: s3_xml::Value(last_modified), + etag: s3_xml::Value(format!("\"{}\"", res.etag)), + }; + let xml = s3_xml::to_xml_with_header(&result)?; + + let mut resp = Response::builder() + .header("Content-Type", "application/xml") + .header("x-amz-version-id", hex::encode(res.version_uuid)) + .header( + "x-amz-copy-source-version-id", + hex::encode(source_version.uuid), + ); + dest_encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(xml))?) +} + +async fn handle_copy_metaonly( + ctx: ReqCtx, + dest_key: &str, + dest_object_meta: ObjectVersionMetaInner, + dest_encryption: EncryptionParams, + source_version: &ObjectVersion, + source_version_data: &ObjectVersionData, + source_version_meta: &ObjectVersionMeta, +) -> Result { + let ReqCtx { + garage, + bucket_id: dest_bucket_id, + .. + } = ctx; // Generate parameters for copied object let new_uuid = gen_uuid(); let new_timestamp = now_msec(); - // Implement x-amz-metadata-directive: REPLACE - let new_meta = match req.headers().get("x-amz-metadata-directive") { - Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => ObjectVersionMeta { - headers: get_headers(req.headers())?, - size: source_version_meta.size, - etag: source_version_meta.etag.clone(), - }, - _ => source_version_meta.clone(), + let new_meta = ObjectVersionMeta { + encryption: dest_encryption.encrypt_meta(dest_object_meta)?, + size: source_version_meta.size, + etag: source_version_meta.etag.clone(), }; - let etag = new_meta.etag.to_string(); + let res = SaveStreamResult { + version_uuid: new_uuid, + version_timestamp: new_timestamp, + etag: new_meta.etag.clone(), + }; // Save object copy match source_version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_meta, bytes) => { + // bytes is either plaintext before&after or encrypted with the + // same keys, so it's ok to just copy it as is let dest_object_version = ObjectVersion { uuid: new_uuid, timestamp: new_timestamp, @@ -87,6 +237,7 @@ pub async fn handle_copy( .get(&source_version.uuid, &EmptyKey) .await?; let source_version = source_version.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; // Write an "uploading" marker in Object table // This holds a reference to the object in the Version table @@ -95,7 +246,8 @@ pub async fn handle_copy( uuid: new_uuid, timestamp: new_timestamp, state: ObjectVersionState::Uploading { - headers: new_meta.headers.clone(), + encryption: new_meta.encryption.clone(), + checksum_algorithm: None, multipart: false, }, }; @@ -162,47 +314,84 @@ pub async fn handle_copy( } } - let last_modified = msec_to_rfc3339(new_timestamp); - let result = CopyObjectResult { - last_modified: s3_xml::Value(last_modified), - etag: s3_xml::Value(format!("\"{}\"", etag)), - }; - let xml = s3_xml::to_xml_with_header(&result)?; - - Ok(Response::builder() - .header("Content-Type", "application/xml") - .header("x-amz-version-id", hex::encode(new_uuid)) - .header( - "x-amz-copy-source-version-id", - hex::encode(source_version.uuid), - ) - .body(Body::from(xml))?) + Ok(res) } +async fn handle_copy_reencrypt( + ctx: ReqCtx, + dest_key: &str, + dest_object_meta: ObjectVersionMetaInner, + dest_encryption: EncryptionParams, + source_version: &ObjectVersion, + source_version_data: &ObjectVersionData, + source_encryption: EncryptionParams, + checksum_mode: ChecksumMode<'_>, +) -> Result { + // basically we will read the source data (decrypt if necessary) + // and save that in a new object (encrypt if necessary), + // by combining the code used in getobject and putobject + let source_stream = full_object_byte_stream( + ctx.garage.clone(), + source_version, + source_version_data, + source_encryption, + ); + + save_stream( + &ctx, + dest_object_meta, + dest_encryption, + source_stream.map_err(|e| Error::from(GarageError::from(e))), + &dest_key.to_string(), + checksum_mode, + ) + .await +} + +// -------- UploadPartCopy --------- + pub async fn handle_upload_part_copy( - garage: Arc, - api_key: &Key, - req: &Request, - dest_bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, dest_key: &str, part_number: u64, upload_id: &str, -) -> Result, Error> { - let copy_precondition = CopyPreconditionHeaders::parse(req)?; +) -> Result, Error> { + let copy_precondition = PreconditionHeaders::parse_copy_source(req)?; let dest_upload_id = multipart::decode_upload_id(upload_id)?; let dest_key = dest_key.to_string(); - let (source_object, (_, _, mut dest_mpu)) = futures::try_join!( - get_copy_source(&garage, api_key, req), - multipart::get_upload(&garage, &dest_bucket_id, &dest_key, &dest_upload_id) + let (source_object, (_, dest_version, mut dest_mpu)) = futures::try_join!( + get_copy_source(&ctx, req), + multipart::get_upload(&ctx, &dest_key, &dest_upload_id) )?; + let ReqCtx { garage, .. } = ctx; + let (source_object_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; // Check precondition on source, e.g. x-amz-copy-source-if-match - copy_precondition.check(source_object_version, &source_version_meta.etag)?; + copy_precondition.check_copy_source(source_object_version, &source_version_meta.etag)?; + + // Determine encryption parameters + let (source_encryption, _) = EncryptionParams::check_decrypt_for_copy_source( + &garage, + req.headers(), + &source_version_meta.encryption, + )?; + let (dest_object_encryption, dest_object_checksum_algorithm) = match dest_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), + _ => unreachable!(), + }; + let (dest_encryption, _) = + EncryptionParams::check_decrypt(&garage, req.headers(), &dest_object_encryption)?; + let same_encryption = EncryptionParams::is_same(&source_encryption, &dest_encryption); // Check source range is valid let source_range = match req.headers().get("x-amz-copy-source-range") { @@ -225,31 +414,29 @@ pub async fn handle_upload_part_copy( }; // Check source version is not inlined - match source_version_data { - ObjectVersionData::DeleteMarker => unreachable!(), - ObjectVersionData::Inline(_meta, _bytes) => { - // This is only for small files, we don't bother handling this. - // (in AWS UploadPartCopy works for parts at least 5MB which - // is never the case of an inline object) - return Err(Error::bad_request( - "Source object is too small (minimum part size is 5Mb)", - )); - } - ObjectVersionData::FirstBlock(_meta, _first_block_hash) => (), - }; + if matches!(source_version_data, ObjectVersionData::Inline(_, _)) { + // This is only for small files, we don't bother handling this. + // (in AWS UploadPartCopy works for parts at least 5MB which + // is never the case of an inline object) + return Err(Error::bad_request( + "Source object is too small (minimum part size is 5Mb)", + )); + } - // Fetch source versin with its block list, - // and destination version to check part hasn't yet been uploaded + // Fetch source version with its block list let source_version = garage .version_table .get(&source_object_version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; // We want to reuse blocks from the source version as much as possible. // However, we still need to get the data from these blocks // because we need to know it to calculate the MD5sum of the part - // which is used as its ETag. + // which is used as its ETag. For encrypted sources or destinations, + // we must always read(+decrypt) and then write(+encrypt), so we + // can never reuse data blocks as is. // First, calculate what blocks we want to keep, // and the subrange of the block to take, if the bounds of the @@ -298,7 +485,9 @@ pub async fn handle_upload_part_copy( dest_mpu_part_key, MpuPart { version: dest_version_id, + // These are all filled in later (bottom of this function) etag: None, + checksum: None, size: None, }, ); @@ -311,32 +500,55 @@ pub async fn handle_upload_part_copy( }, false, ); + // write an empty version now to be the parent of the block_ref entries + garage.version_table.insert(&dest_version).await?; // Now, actually copy the blocks - let mut md5hasher = Md5::new(); + let mut checksummer = Checksummer::init(&Default::default(), !dest_encryption.is_encrypted()) + .add(dest_object_checksum_algorithm); // First, create a stream that is able to read the source blocks // and extract the subrange if necessary. // The second returned value is an Option, that is Some // if and only if the block returned is a block that already existed - // in the Garage data store (thus we don't need to save it again). + // in the Garage data store and can be reused as-is instead of having + // to save it again. This excludes encrypted source blocks that we had + // to decrypt. let garage2 = garage.clone(); let order_stream = OrderTag::stream(); let source_blocks = stream::iter(blocks_to_copy) .enumerate() - .flat_map(|(i, (block_hash, range_to_copy))| { + .map(|(i, (block_hash, range_to_copy))| { let garage3 = garage2.clone(); - stream::once(async move { - let data = garage3 - .block_manager - .rpc_get_block(&block_hash, Some(order_stream.order(i as u64))) + async move { + let stream = source_encryption + .get_block(&garage3, &block_hash, Some(order_stream.order(i as u64))) .await?; + let data = read_stream_to_end(stream).await?.into_bytes(); + // For each item, we return a tuple of: + // 1. the full data block (decrypted) + // 2. an Option that indicates the hash of the block in the block store, + // only if it can be re-used as-is in the copied object match range_to_copy { - Some(r) => Ok((data.slice(r), None)), - None => Ok((data, Some(block_hash))), + Some(r) => { + // If we are taking a subslice of the data, we cannot reuse the block as-is + Ok((data.slice(r), None)) + } + None if same_encryption => { + // If the data is unencrypted before & after, or if we are using + // the same encryption key, we can reuse the stored block, no need + // to re-send it to storage nodes. + Ok((data, Some(block_hash))) + } + None => { + // If we are decrypting / (re)encrypting with different keys, + // we cannot reuse the block as-is + Ok((data, None)) + } } - }) + } }) + .buffered(2) .peekable(); // The defragmenter is a custom stream (defined below) that concatenates @@ -344,69 +556,106 @@ pub async fn handle_upload_part_copy( // It returns a series of (Vec, Option). // When it is done, it returns an empty vec. // Same as the previous iterator, the Option is Some(_) if and only if - // it's an existing block of the Garage data store. + // it's an existing block of the Garage data store that can be reused. let mut defragmenter = Defragmenter::new(garage.config.block_size, Box::pin(source_blocks)); let mut current_offset = 0; let mut next_block = defragmenter.next().await?; + let mut blocks_to_dup = dest_version.clone(); + // TODO this could be optimized similarly to read_and_put_blocks + // low priority because uploadpartcopy is rarely used loop { let (data, existing_block_hash) = next_block; if data.is_empty() { break; } - md5hasher.update(&data[..]); + let data_len = data.len() as u64; - let must_upload = existing_block_hash.is_none(); - let final_hash = existing_block_hash.unwrap_or_else(|| blake2sum(&data[..])); + let (checksummer_updated, (data_to_upload, final_hash)) = + tokio::task::spawn_blocking(move || { + checksummer.update(&data[..]); - dest_version.blocks.clear(); - dest_version.blocks.put( + let tup = match existing_block_hash { + Some(hash) if same_encryption => (None, hash), + _ => { + let data_enc = dest_encryption.encrypt_block(data)?; + let hash = blake2sum(&data_enc); + (Some(data_enc), hash) + } + }; + Ok::<_, Error>((checksummer, tup)) + }) + .await + .unwrap()?; + checksummer = checksummer_updated; + + let (version_block_key, version_block) = ( VersionBlockKey { part_number, offset: current_offset, }, VersionBlock { hash: final_hash, - size: data.len() as u64, + size: data_len, }, ); - current_offset += data.len() as u64; + current_offset += data_len; - let block_ref = BlockRef { - block: final_hash, - version: dest_version_id, - deleted: false.into(), - }; - - let garage2 = garage.clone(); - let res = futures::try_join!( - // Thing 1: if the block is not exactly a block that existed before, - // we need to insert that data as a new block. - async move { - if must_upload { - garage2.block_manager.rpc_put_block(final_hash, data).await - } else { - Ok(()) - } - }, - async { + let next = if let Some(final_data) = data_to_upload { + dest_version.blocks.clear(); + dest_version.blocks.put(version_block_key, version_block); + let block_ref = BlockRef { + block: final_hash, + version: dest_version_id, + deleted: false.into(), + }; + let (_, _, _, next) = futures::try_join!( + // Thing 1: if the block is not exactly a block that existed before, + // we need to insert that data as a new block. + garage.block_manager.rpc_put_block( + final_hash, + final_data, + dest_encryption.is_encrypted(), + None + ), // Thing 2: we need to insert the block in the version - garage.version_table.insert(&dest_version).await?; + garage.version_table.insert(&dest_version), // Thing 3: we need to add a block reference - garage.block_ref_table.insert(&block_ref).await - }, - // Thing 4: we need to prefetch the next block - defragmenter.next(), - )?; - next_block = res.2; + garage.block_ref_table.insert(&block_ref), + // Thing 4: we need to read the next block + defragmenter.next(), + )?; + next + } else { + blocks_to_dup.blocks.put(version_block_key, version_block); + defragmenter.next().await? + }; + next_block = next; } assert_eq!(current_offset, source_range.length); - let data_md5sum = md5hasher.finalize(); - let etag = hex::encode(data_md5sum); + // Put the duplicated blocks into the version & block_refs tables + let block_refs_to_put = blocks_to_dup + .blocks + .items() + .iter() + .map(|b| BlockRef { + block: b.1.hash, + version: dest_version_id, + deleted: false.into(), + }) + .collect::>(); + futures::try_join!( + garage.version_table.insert(&blocks_to_dup), + garage.block_ref_table.insert_many(&block_refs_to_put[..]), + )?; + + let checksums = checksummer.finalize(); + let etag = dest_encryption.etag_from_md5(&checksums.md5); + let checksum = checksums.extract(dest_object_checksum_algorithm); // Put the part's ETag in the Versiontable dest_mpu.parts.put( @@ -414,6 +663,7 @@ pub async fn handle_upload_part_copy( MpuPart { version: dest_version_id, etag: Some(etag.clone()), + checksum, size: Some(current_offset), }, ); @@ -426,20 +676,21 @@ pub async fn handle_upload_part_copy( last_modified: s3_xml::Value(msec_to_rfc3339(source_object_version.timestamp)), })?; - Ok(Response::builder() + let mut resp = Response::builder() .header("Content-Type", "application/xml") .header( "x-amz-copy-source-version-id", hex::encode(source_object_version.uuid), - ) - .body(Body::from(resp_xml))?) + ); + dest_encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(resp_xml))?) } -async fn get_copy_source( - garage: &Garage, - api_key: &Key, - req: &Request, -) -> Result { +async fn get_copy_source(ctx: &ReqCtx, req: &Request) -> Result { + let ReqCtx { + garage, api_key, .. + } = ctx; + let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?; @@ -447,7 +698,8 @@ async fn get_copy_source( let source_bucket_id = garage .bucket_helper() .resolve_bucket(&source_bucket.to_string(), api_key) - .await?; + .await + .map_err(pass_helper_error)?; if !api_key.allow_read(&source_bucket_id) { return Err(Error::forbidden(format!( @@ -493,97 +745,6 @@ fn extract_source_info( Ok((source_version, source_version_data, source_version_meta)) } -struct CopyPreconditionHeaders { - copy_source_if_match: Option>, - copy_source_if_modified_since: Option, - copy_source_if_none_match: Option>, - copy_source_if_unmodified_since: Option, -} - -impl CopyPreconditionHeaders { - fn parse(req: &Request) -> Result { - Ok(Self { - copy_source_if_match: req - .headers() - .get("x-amz-copy-source-if-match") - .map(|x| x.to_str()) - .transpose()? - .map(|x| { - x.split(',') - .map(|m| m.trim().trim_matches('"').to_string()) - .collect::>() - }), - copy_source_if_modified_since: req - .headers() - .get("x-amz-copy-source-if-modified-since") - .map(|x| x.to_str()) - .transpose()? - .map(httpdate::parse_http_date) - .transpose() - .ok_or_bad_request("Invalid date in x-amz-copy-source-if-modified-since")?, - copy_source_if_none_match: req - .headers() - .get("x-amz-copy-source-if-none-match") - .map(|x| x.to_str()) - .transpose()? - .map(|x| { - x.split(',') - .map(|m| m.trim().trim_matches('"').to_string()) - .collect::>() - }), - copy_source_if_unmodified_since: req - .headers() - .get("x-amz-copy-source-if-unmodified-since") - .map(|x| x.to_str()) - .transpose()? - .map(httpdate::parse_http_date) - .transpose() - .ok_or_bad_request("Invalid date in x-amz-copy-source-if-unmodified-since")?, - }) - } - - fn check(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> { - let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp); - - let ok = match ( - &self.copy_source_if_match, - &self.copy_source_if_unmodified_since, - &self.copy_source_if_none_match, - &self.copy_source_if_modified_since, - ) { - // TODO I'm not sure all of the conditions are evaluated correctly here - - // If we have both if-match and if-unmodified-since, - // basically we don't care about if-unmodified-since, - // because in the spec it says that if if-match evaluates to - // true but if-unmodified-since evaluates to false, - // the copy is still done. - (Some(im), _, None, None) => im.iter().any(|x| x == etag || x == "*"), - (None, Some(ius), None, None) => v_date <= *ius, - - // If we have both if-none-match and if-modified-since, - // then both of the two conditions must evaluate to true - (None, None, Some(inm), Some(ims)) => { - !inm.iter().any(|x| x == etag || x == "*") && v_date > *ims - } - (None, None, Some(inm), None) => !inm.iter().any(|x| x == etag || x == "*"), - (None, None, None, Some(ims)) => v_date > *ims, - (None, None, None, None) => true, - _ => { - return Err(Error::bad_request( - "Invalid combination of x-amz-copy-source-if-xxxxx headers", - )) - } - }; - - if ok { - Ok(()) - } else { - Err(Error::PreconditionFailed) - } - } -} - type BlockStreamItemOk = (Bytes, Option); type BlockStreamItem = Result; @@ -653,7 +814,7 @@ pub struct CopyPartResult { #[cfg(test)] mod tests { use super::*; - use crate::s3::xml::to_xml_with_header; + use crate::xml::to_xml_with_header; #[test] fn copy_object_result() -> Result<(), Error> { diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 49097ad1..1f365beb 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -1,28 +1,20 @@ use quick_xml::de::from_reader; -use std::sync::Arc; -use http::header::{ - ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, - ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, -}; -use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode}; +use hyper::{header::HeaderName, Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; - use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; -use garage_util::data::*; -pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +use garage_api_common::helpers::*; - if let Some(cors) = param.cors_config.get() { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; + +pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(cors) = bucket_params.cors_config.get() { let wc = CorsConfiguration { xmlns: (), cors_rules: cors @@ -34,202 +26,59 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } else { Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } } -pub async fn handle_delete_cors( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.cors_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.cors_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_put_cors( - garage: Arc, - mut bucket: Bucket, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let body = req.into_body().collect().await?; let conf: CorsConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .cors_config .update(Some(conf.into_garage_cors_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) - .body(Body::empty())?) -} - -pub async fn handle_options_s3api( - garage: Arc, - req: &Request, - bucket_name: Option, -) -> Result, Error> { - // FIXME: CORS rules of buckets with local aliases are - // not taken into account. - - // If the bucket name is a global bucket name, - // we try to apply the CORS rules of that bucket. - // If a user has a local bucket name that has - // the same name, its CORS rules won't be applied - // and will be shadowed by the rules of the globally - // existing bucket (but this is inevitable because - // OPTIONS calls are not auhtenticated). - if let Some(bn) = bucket_name { - let helper = garage.bucket_helper(); - let bucket_id = helper.resolve_global_bucket_name(&bn).await?; - if let Some(id) = bucket_id { - let bucket = garage.bucket_helper().get_existing_bucket(id).await?; - handle_options_for_bucket(req, &bucket) - } else { - // If there is a bucket name in the request, but that name - // does not correspond to a global alias for a bucket, - // then it's either a non-existing bucket or a local bucket. - // We have no way of knowing, because the request is not - // authenticated and thus we can't resolve local aliases. - // We take the permissive approach of allowing everything, - // because we don't want to prevent web apps that use - // local bucket names from making API calls. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "*") - .status(StatusCode::OK) - .body(Body::empty())?) - } - } else { - // If there is no bucket name in the request, - // we are doing a ListBuckets call, which we want to allow - // for all origins. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") - .status(StatusCode::OK) - .body(Body::empty())?) - } -} - -pub fn handle_options_for_bucket( - req: &Request, - bucket: &Bucket, -) -> Result, Error> { - let origin = req - .headers() - .get("Origin") - .ok_or_bad_request("Missing Origin header")? - .to_str()?; - let request_method = req - .headers() - .get(ACCESS_CONTROL_REQUEST_METHOD) - .ok_or_bad_request("Missing Access-Control-Request-Method header")? - .to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { - let matching_rule = cors_config - .iter() - .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); - if let Some(rule) = matching_rule { - let mut resp = Response::builder() - .status(StatusCode::OK) - .body(Body::empty())?; - add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; - return Ok(resp); - } - } - - Err(Error::forbidden("This CORS request is not allowed.")) -} - -pub fn find_matching_cors_rule<'a>( - bucket: &'a Bucket, - req: &Request, -) -> Result, Error> { - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { - if let Some(origin) = req.headers().get("Origin") { - let origin = origin.to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - return Ok(cors_config.iter().find(|rule| { - cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) - })); - } - } - Ok(None) -} - -fn cors_rule_matches<'a, HI, S>( - rule: &GarageCorsRule, - origin: &'a str, - method: &'a str, - mut request_headers: HI, -) -> bool -where - HI: Iterator, - S: AsRef, -{ - rule.allow_origins.iter().any(|x| x == "*" || x == origin) - && rule.allow_methods.iter().any(|x| x == "*" || x == method) - && request_headers.all(|h| { - rule.allow_headers - .iter() - .any(|x| x == "*" || x == h.as_ref()) - }) -} - -pub fn add_cors_headers( - resp: &mut Response, - rule: &GarageCorsRule, -) -> Result<(), http::header::InvalidHeaderValue> { - let h = resp.headers_mut(); - h.insert( - ACCESS_CONTROL_ALLOW_ORIGIN, - rule.allow_origins.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_METHODS, - rule.allow_methods.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_HEADERS, - rule.allow_headers.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_EXPOSE_HEADERS, - rule.expose_headers.join(", ").parse()?, - ); - Ok(()) + .body(empty_body())?) } // ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- @@ -239,7 +88,9 @@ pub fn add_cors_headers( pub struct CorsConfiguration { #[serde(serialize_with = "xmlns_tag", skip_deserializing)] pub xmlns: (), - #[serde(rename = "CORSRule")] + // "default" is required to be able to parse an empty list of rules, + // cf https://docs.rs/quick-xml/latest/quick_xml/de/#sequences-xsall-and-xssequence-xml-schema-types + #[serde(rename = "CORSRule", default)] pub cors_rules: Vec, } @@ -421,4 +272,26 @@ mod tests { Ok(()) } + + #[test] + fn test_deserialize_norules() -> Result<(), Error> { + let message = r#" +"#; + let conf: CorsConfiguration = from_str(message).unwrap(); + let ref_value = CorsConfiguration { + xmlns: (), + cors_rules: vec![], + }; + assert_eq! { + ref_value, + conf + }; + + let message2 = to_xml_with_header(&ref_value)?; + + let cleanup = |c: &str| c.replace(char::is_whitespace, ""); + assert_eq!(cleanup(message), cleanup(&message2)); + + Ok(()) + } } diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index b337155f..d785b9d8 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -1,89 +1,73 @@ -use std::sync::Arc; - -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use garage_util::data::*; -use garage_util::time::*; -use garage_model::garage::Garage; use garage_model::s3::object_table::*; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::helpers::*; -async fn handle_delete_internal( - garage: &Garage, - bucket_id: Uuid, - key: &str, -) -> Result<(Uuid, Uuid), Error> { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::put::next_timestamp; +use crate::xml as s3_xml; + +async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let object = garage .object_table - .get(&bucket_id, &key.to_string()) + .get(bucket_id, &key.to_string()) .await? .ok_or(Error::NoSuchKey)?; // No need to delete - let interesting_versions = object.versions().iter().filter(|v| { - !matches!( - v.state, - ObjectVersionState::Aborted - | ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) - ) - }); + let del_timestamp = next_timestamp(Some(&object)); + let del_uuid = gen_uuid(); - let mut version_to_delete = None; - let mut timestamp = now_msec(); - for v in interesting_versions { - if v.timestamp + 1 > timestamp || version_to_delete.is_none() { - version_to_delete = Some(v.uuid); + let deleted_version = object + .versions() + .iter() + .rev() + .find(|v| !matches!(&v.state, ObjectVersionState::Aborted)) + .or_else(|| object.versions().iter().rev().next()); + let deleted_version = match deleted_version { + Some(dv) => dv.uuid, + None => { + warn!("Object has no versions: {:?}", object); + Uuid::from([0u8; 32]) } - timestamp = std::cmp::max(timestamp, v.timestamp + 1); - } - - let deleted_version = version_to_delete.ok_or(Error::NoSuchKey)?; - - let version_uuid = gen_uuid(); + }; let object = Object::new( - bucket_id, + *bucket_id, key.into(), vec![ObjectVersion { - uuid: version_uuid, - timestamp, + uuid: del_uuid, + timestamp: del_timestamp, state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker), }], ); garage.object_table.insert(&object).await?; - Ok((deleted_version, version_uuid)) + Ok((deleted_version, del_uuid)) } -pub async fn handle_delete( - garage: Arc, - bucket_id: Uuid, - key: &str, -) -> Result, Error> { - match handle_delete_internal(&garage, bucket_id, key).await { +pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result, Error> { + match handle_delete_internal(&ctx, key).await { Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::from(vec![])) + .body(empty_body()) .unwrap()), Err(e) => Err(e), } } pub async fn handle_delete_objects( - garage: Arc, - bucket_id: Uuid, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let body = req.into_body().collect().await?; let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?; let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?; @@ -92,7 +76,7 @@ pub async fn handle_delete_objects( let mut ret_errors = Vec::new(); for obj in cmd.objects.iter() { - match handle_delete_internal(&garage, bucket_id, &obj.key).await { + match handle_delete_internal(&ctx, &obj.key).await { Ok((deleted_version, delete_marker_version)) => { if cmd.quiet { continue; @@ -122,7 +106,7 @@ pub async fn handle_delete_objects( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } struct DeleteRequest { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs new file mode 100644 index 00000000..fa7285ca --- /dev/null +++ b/src/api/s3/encryption.rs @@ -0,0 +1,596 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::pin::Pin; + +use aes_gcm::{ + aead::stream::{DecryptorLE31, EncryptorLE31, StreamLE31}, + aead::{Aead, AeadCore, KeyInit, OsRng}, + aes::cipher::crypto_common::rand_core::RngCore, + aes::cipher::typenum::Unsigned, + Aes256Gcm, Key, Nonce, +}; +use base64::prelude::*; +use bytes::Bytes; + +use futures::stream::Stream; +use futures::task; +use tokio::io::BufReader; + +use http::header::{HeaderMap, HeaderName, HeaderValue}; + +use garage_net::bytes_buf::BytesBuf; +use garage_net::stream::{stream_asyncread, ByteStream}; +use garage_rpc::rpc_helper::OrderTag; +use garage_util::data::Hash; +use garage_util::error::Error as GarageError; +use garage_util::migrate::Migrate; + +use garage_model::garage::Garage; +use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; + +use garage_api_common::common_error::*; +use garage_api_common::signature::checksum::Md5Checksum; + +use crate::error::Error; + +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-key"); +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5"); + +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-algorithm"); +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-key"); +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-key-md5"); + +const CUSTOMER_ALGORITHM_AES256: &[u8] = b"AES256"; + +type Md5Output = md5::digest::Output; + +type StreamNonceSize = aes_gcm::aead::stream::NonceSize>; + +// Data blocks are encrypted by smaller chunks of size 4096 bytes, +// so that data can be streamed when reading. +// This size has to be known and has to be constant, or data won't be +// readable anymore. DO NOT CHANGE THIS VALUE. +const STREAM_ENC_PLAIN_CHUNK_SIZE: usize = 0x1000; // 4096 bytes +const STREAM_ENC_CYPER_CHUNK_SIZE: usize = STREAM_ENC_PLAIN_CHUNK_SIZE + 16; + +#[derive(Clone, Copy)] +pub enum EncryptionParams { + Plaintext, + SseC { + client_key: Key, + client_key_md5: Md5Output, + compression_level: Option, + }, +} + +impl EncryptionParams { + pub fn is_encrypted(&self) -> bool { + !matches!(self, Self::Plaintext) + } + + pub fn is_same(a: &Self, b: &Self) -> bool { + let relevant_info = |x: &Self| match x { + Self::Plaintext => None, + Self::SseC { + client_key, + compression_level, + .. + } => Some((*client_key, compression_level.is_some())), + }; + relevant_info(a) == relevant_info(b) + } + + pub fn new_from_headers( + garage: &Garage, + headers: &HeaderMap, + ) -> Result { + let key = parse_request_headers( + headers, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + match key { + Some((client_key, client_key_md5)) => Ok(EncryptionParams::SseC { + client_key, + client_key_md5, + compression_level: garage.config.compression_level, + }), + None => Ok(EncryptionParams::Plaintext), + } + } + + pub fn add_response_headers(&self, resp: &mut http::response::Builder) { + if let Self::SseC { client_key_md5, .. } = self { + let md5 = BASE64_STANDARD.encode(&client_key_md5); + + resp.headers_mut().unwrap().insert( + X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + HeaderValue::from_bytes(CUSTOMER_ALGORITHM_AES256).unwrap(), + ); + resp.headers_mut().unwrap().insert( + X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + HeaderValue::from_bytes(md5.as_bytes()).unwrap(), + ); + } + } + + pub fn check_decrypt<'a>( + garage: &Garage, + headers: &HeaderMap, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + let key = parse_request_headers( + headers, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + Self::check_decrypt_common(garage, key, obj_enc) + } + + pub fn check_decrypt_for_copy_source<'a>( + garage: &Garage, + headers: &HeaderMap, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + let key = parse_request_headers( + headers, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + Self::check_decrypt_common(garage, key, obj_enc) + } + + fn check_decrypt_common<'a>( + garage: &Garage, + key: Option<(Key, Md5Output)>, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + match (key, &obj_enc) { + ( + Some((client_key, client_key_md5)), + ObjectVersionEncryption::SseC { inner, compressed }, + ) => { + let enc = Self::SseC { + client_key, + client_key_md5, + compression_level: if *compressed { + Some(garage.config.compression_level.unwrap_or(1)) + } else { + None + }, + }; + let plaintext = enc.decrypt_blob(&inner)?; + let inner = ObjectVersionMetaInner::decode(&plaintext) + .ok_or_internal_error("Could not decode encrypted metadata")?; + Ok((enc, Cow::Owned(inner))) + } + (None, ObjectVersionEncryption::Plaintext { inner }) => { + Ok((Self::Plaintext, Cow::Borrowed(inner))) + } + (_, ObjectVersionEncryption::SseC { .. }) => { + Err(Error::bad_request("Object is encrypted")) + } + (Some(_), _) => { + // TODO: should this be an OK scenario? + Err(Error::bad_request("Trying to decrypt a plaintext object")) + } + } + } + + pub fn encrypt_meta( + &self, + meta: ObjectVersionMetaInner, + ) -> Result { + match self { + Self::SseC { + compression_level, .. + } => { + let plaintext = meta.encode().map_err(GarageError::from)?; + let ciphertext = self.encrypt_blob(&plaintext)?; + Ok(ObjectVersionEncryption::SseC { + inner: ciphertext.into_owned(), + compressed: compression_level.is_some(), + }) + } + Self::Plaintext => Ok(ObjectVersionEncryption::Plaintext { inner: meta }), + } + } + + // ---- generating object Etag values ---- + pub fn etag_from_md5(&self, md5sum: &Option) -> String { + match self { + Self::Plaintext => md5sum + .map(|x| hex::encode(&x[..])) + .expect("md5 digest should have been computed"), + Self::SseC { .. } => { + // AWS specifies that for encrypted objects, the Etag is not + // the md5sum of the data, but doesn't say what it is. + // So we just put some random bytes. + let mut random = [0u8; 16]; + OsRng.fill_bytes(&mut random); + hex::encode(&random) + } + } + } + + // ---- generic function for encrypting / decrypting blobs ---- + // Prepends a randomly-generated nonce to the encrypted value. + // This is used for encrypting object metadata and inlined data for small objects. + // This does not compress anything. + + pub fn encrypt_blob<'a>(&self, blob: &'a [u8]) -> Result, Error> { + match self { + Self::SseC { client_key, .. } => { + let cipher = Aes256Gcm::new(&client_key); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ciphertext = cipher + .encrypt(&nonce, blob) + .ok_or_internal_error("Encryption failed")?; + Ok(Cow::Owned([nonce.to_vec(), ciphertext].concat())) + } + Self::Plaintext => Ok(Cow::Borrowed(blob)), + } + } + + pub fn decrypt_blob<'a>(&self, blob: &'a [u8]) -> Result, Error> { + match self { + Self::SseC { client_key, .. } => { + let cipher = Aes256Gcm::new(&client_key); + let nonce_size = ::NonceSize::to_usize(); + let nonce = Nonce::from_slice( + blob.get(..nonce_size) + .ok_or_internal_error("invalid encrypted data")?, + ); + let plaintext = cipher + .decrypt(nonce, &blob[nonce_size..]) + .ok_or_bad_request( + "Invalid encryption key, could not decrypt object metadata.", + )?; + Ok(Cow::Owned(plaintext)) + } + Self::Plaintext => Ok(Cow::Borrowed(blob)), + } + } + + // ---- function for encrypting / decrypting byte streams ---- + + /// Get a data block from the storage node, and decrypt+decompress it + /// if necessary. If object is plaintext, just get it without any processing. + pub async fn get_block( + &self, + garage: &Garage, + hash: &Hash, + order: Option, + ) -> Result { + let raw_block = garage + .block_manager + .rpc_get_block_streaming(hash, order) + .await?; + Ok(self.decrypt_block_stream(raw_block)) + } + + pub fn decrypt_block_stream(&self, stream: ByteStream) -> ByteStream { + match self { + Self::Plaintext => stream, + Self::SseC { + client_key, + compression_level, + .. + } => { + let plaintext = DecryptStream::new(stream, *client_key); + if compression_level.is_some() { + let reader = stream_asyncread(Box::pin(plaintext)); + let reader = BufReader::new(reader); + let reader = async_compression::tokio::bufread::ZstdDecoder::new(reader); + Box::pin(tokio_util::io::ReaderStream::new(reader)) + } else { + Box::pin(plaintext) + } + } + } + } + + /// Encrypt a data block if encryption is set, for use before + /// putting the data blocks into storage + pub fn encrypt_block(&self, block: Bytes) -> Result { + match self { + Self::Plaintext => Ok(block), + Self::SseC { + client_key, + compression_level, + .. + } => { + let block = if let Some(level) = compression_level { + Cow::Owned( + garage_block::zstd_encode(block.as_ref(), *level) + .ok_or_internal_error("failed to compress data block")?, + ) + } else { + Cow::Borrowed(block.as_ref()) + }; + + let mut ret = Vec::with_capacity(block.len() + 32 + block.len() / 64); + + let mut nonce: Nonce = Default::default(); + OsRng.fill_bytes(&mut nonce); + ret.extend_from_slice(nonce.as_slice()); + + let mut cipher = EncryptorLE31::::new(&client_key, &nonce); + let mut iter = block.chunks(STREAM_ENC_PLAIN_CHUNK_SIZE).peekable(); + + if iter.peek().is_none() { + // Empty stream: we encrypt an empty last chunk + let chunk_enc = cipher + .encrypt_last(&[][..]) + .ok_or_internal_error("failed to encrypt chunk")?; + ret.extend_from_slice(&chunk_enc); + } else { + loop { + let chunk = iter.next().unwrap(); + if iter.peek().is_some() { + let chunk_enc = cipher + .encrypt_next(chunk) + .ok_or_internal_error("failed to encrypt chunk")?; + assert_eq!(chunk.len(), STREAM_ENC_PLAIN_CHUNK_SIZE); + assert_eq!(chunk_enc.len(), STREAM_ENC_CYPER_CHUNK_SIZE); + ret.extend_from_slice(&chunk_enc); + } else { + // use encrypt_last for the last chunk + let chunk_enc = cipher + .encrypt_last(chunk) + .ok_or_internal_error("failed to encrypt chunk")?; + ret.extend_from_slice(&chunk_enc); + break; + } + } + } + + Ok(ret.into()) + } + } + } +} + +fn parse_request_headers( + headers: &HeaderMap, + alg_header: &HeaderName, + key_header: &HeaderName, + md5_header: &HeaderName, +) -> Result, Md5Output)>, Error> { + let alg = headers.get(alg_header).map(HeaderValue::as_bytes); + let key = headers.get(key_header).map(HeaderValue::as_bytes); + let md5 = headers.get(md5_header).map(HeaderValue::as_bytes); + + match alg { + Some(CUSTOMER_ALGORITHM_AES256) => { + use md5::{Digest, Md5}; + + let key_b64 = + key.ok_or_bad_request("Missing server-side-encryption-customer-key header")?; + let key_bytes: [u8; 32] = BASE64_STANDARD + .decode(&key_b64) + .ok_or_bad_request( + "Invalid server-side-encryption-customer-key header: invalid base64", + )? + .try_into() + .ok() + .ok_or_bad_request( + "Invalid server-side-encryption-customer-key header: invalid length", + )?; + + let md5_b64 = + md5.ok_or_bad_request("Missing server-side-encryption-customer-key-md5 header")?; + let md5_bytes = BASE64_STANDARD.decode(&md5_b64).ok_or_bad_request( + "Invalid server-side-encryption-customer-key-md5 header: invalid bass64", + )?; + + let mut hasher = Md5::new(); + hasher.update(&key_bytes[..]); + let our_md5 = hasher.finalize(); + if our_md5.as_slice() != md5_bytes.as_slice() { + return Err(Error::bad_request( + "Server-side encryption client key MD5 checksum does not match", + )); + } + + Ok(Some((key_bytes.into(), our_md5))) + } + Some(alg) => Err(Error::InvalidEncryptionAlgorithm( + String::from_utf8_lossy(alg).into_owned(), + )), + None => { + if key.is_some() || md5.is_some() { + Err(Error::bad_request( + "Unexpected server-side-encryption-customer-key{,-md5} header(s)", + )) + } else { + Ok(None) + } + } + } +} + +// ---- encrypt & decrypt streams ---- + +#[pin_project::pin_project] +struct DecryptStream { + #[pin] + stream: ByteStream, + done_reading: bool, + buf: BytesBuf, + key: Key, + state: DecryptStreamState, +} + +enum DecryptStreamState { + Starting, + Running(DecryptorLE31), + Done, +} + +impl DecryptStream { + fn new(stream: ByteStream, key: Key) -> Self { + Self { + stream, + done_reading: false, + buf: BytesBuf::new(), + key, + state: DecryptStreamState::Starting, + } + } +} + +impl Stream for DecryptStream { + type Item = Result; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll> { + use std::task::Poll; + + let mut this = self.project(); + + // The first bytes of the stream should contain the starting nonce. + // If we don't have a Running state, it means that we haven't + // yet read the nonce. + while matches!(this.state, DecryptStreamState::Starting) { + let nonce_size = StreamNonceSize::to_usize(); + if let Some(nonce) = this.buf.take_exact(nonce_size) { + let nonce = Nonce::from_slice(nonce.as_ref()); + *this.state = DecryptStreamState::Running(DecryptorLE31::new(&this.key, nonce)); + break; + } + + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(e))); + } + None => { + return Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "Decrypt: unexpected EOF, could not read nonce", + )))); + } + } + } + + // Read at least one byte more than the encrypted chunk size + // (if possible), so that we know if we are decrypting the + // last chunk or not. + while !*this.done_reading && this.buf.len() <= STREAM_ENC_CYPER_CHUNK_SIZE { + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(e))); + } + None => { + *this.done_reading = true; + break; + } + } + } + + if matches!(this.state, DecryptStreamState::Done) { + if !this.buf.is_empty() { + return Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Decrypt: unexpected bytes after last encrypted chunk", + )))); + } + return Poll::Ready(None); + } + + let res = if this.buf.len() > STREAM_ENC_CYPER_CHUNK_SIZE { + // we have strictly more bytes than the encrypted chunk size, + // so we know this is not the last + let DecryptStreamState::Running(ref mut cipher) = this.state else { + unreachable!() + }; + let chunk = this.buf.take_exact(STREAM_ENC_CYPER_CHUNK_SIZE).unwrap(); + let chunk_dec = cipher.decrypt_next(chunk.as_ref()); + if let Ok(c) = &chunk_dec { + assert_eq!(c.len(), STREAM_ENC_PLAIN_CHUNK_SIZE); + } + chunk_dec + } else { + // We have one encrypted chunk size or less, even though we tried + // to read more, so this is the last chunk. Decrypt using the + // appropriate decrypt_last() function that then destroys the cipher. + let state = std::mem::replace(this.state, DecryptStreamState::Done); + let DecryptStreamState::Running(cipher) = state else { + unreachable!() + }; + let chunk = this.buf.take_all(); + cipher.decrypt_last(chunk.as_ref()) + }; + + match res { + Ok(bytes) if bytes.is_empty() => Poll::Ready(None), + Ok(bytes) => Poll::Ready(Some(Ok(bytes.into()))), + Err(_) => Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Decryption failed", + )))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use futures::stream::StreamExt; + use garage_net::stream::read_stream_to_end; + + fn stream() -> ByteStream { + Box::pin( + futures::stream::iter(16usize..1024) + .map(|i| Ok(Bytes::from(vec![(i % 256) as u8; (i * 37) % 1024]))), + ) + } + + async fn test_block_enc(compression_level: Option) { + let enc = EncryptionParams::SseC { + client_key: Aes256Gcm::generate_key(&mut OsRng), + client_key_md5: Default::default(), // not needed + compression_level, + }; + + let block_plain = read_stream_to_end(stream()).await.unwrap().into_bytes(); + + let block_enc = enc.encrypt_block(block_plain.clone()).unwrap(); + + let block_dec = + enc.decrypt_block_stream(Box::pin(futures::stream::once(async { Ok(block_enc) }))); + let block_dec = read_stream_to_end(block_dec).await.unwrap().into_bytes(); + + assert_eq!(block_plain, block_dec); + assert!(block_dec.len() > 128000); + } + + #[tokio::test] + async fn test_encrypt_block() { + test_block_enc(None).await + } + + #[tokio::test] + async fn test_encrypt_block_compressed() { + test_block_enc(Some(1)).await + } +} diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index c50cff9f..64112084 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -1,100 +1,107 @@ use std::convert::TryInto; -use err_derive::Error; use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; use garage_model::helper::error::Error as HelperError; -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::s3::xml as s3_xml; -use crate::signature::error::Error as SignatureError; +pub(crate) use garage_api_common::common_error::pass_helper_error; + +use garage_api_common::common_error::{ + commonErrorDerivative, helper_error_as_internal, CommonError, +}; + +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; + +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; + +use crate::xml as s3_xml; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// The multipart upload requested don't exists - #[error(display = "Upload not found")] + #[error("Upload not found")] NoSuchUpload, /// Precondition failed (e.g. x-amz-copy-source-if-match) - #[error(display = "At least one of the preconditions you specified did not hold")] + #[error("At least one of the preconditions you specified did not hold")] PreconditionFailed, /// Parts specified in CMU request do not match parts actually uploaded - #[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")] + #[error("Parts given to CompleteMultipartUpload do not match uploaded parts")] InvalidPart, /// Parts given to CompleteMultipartUpload were not in ascending order - #[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")] + #[error("Parts given to CompleteMultipartUpload were not in ascending order")] InvalidPartOrder, /// In CompleteMultipartUpload: not enough data /// (here we are more lenient than AWS S3) - #[error(display = "Proposed upload is smaller than the minimum allowed object size")] + #[error("Proposed upload is smaller than the minimum allowed object size")] EntityTooSmall, // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), /// The request used an invalid path - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8String(#[error(source)] std::string::FromUtf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8String(#[from] std::string::FromUtf8Error), /// The client sent invalid XML data - #[error(display = "Invalid XML: {}", _0)] + #[error("Invalid XML: {0}")] InvalidXml(String), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// The client sent a range header with invalid value + #[error("Invalid HTTP range: {0:?}")] + InvalidRange((http_range::HttpRangeParseError, u64)), /// The client sent a range header with invalid value - #[error(display = "Invalid HTTP range: {:?}", _0)] - InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)), + #[error("Invalid encryption algorithm: {0:?}, should be AES256")] + InvalidEncryptionAlgorithm(String), + + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), /// The client sent a request for an action not supported by garage - #[error(display = "Unimplemented action: {}", _0)] + #[error("Unimplemented action: {0}")] NotImplemented(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) +commonErrorDerivative!(Error); + +// Helper errors are always passed as internal errors by default. +// To pass the specific error code back to the client, use `pass_helper_error`. +impl From for Error { + fn from(err: HelperError) -> Error { + Error::Common(helper_error_as_internal(err)) } } -impl CommonErrorDerivative for Error {} - -impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - e => Self::bad_request(format!("{}", e)), - } +impl From<(http_range::HttpRangeParseError, u64)> for Error { + fn from(err: (http_range::HttpRangeParseError, u64)) -> Error { + Error::InvalidRange(err) } } @@ -118,7 +125,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), - SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } @@ -143,9 +150,9 @@ impl Error { Error::NotImplemented(_) => "NotImplemented", Error::InvalidXml(_) => "MalformedXML", Error::InvalidRange(_) => "InvalidRange", - Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) | Error::InvalidHeader(_) => { - "InvalidRequest" - } + Error::InvalidDigest(_) => "InvalidDigest", + Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) => "InvalidRequest", + Error::InvalidEncryptionAlgorithm(_) => "InvalidEncryptionAlgorithmError", } } } @@ -163,10 +170,11 @@ impl ApiError for Error { | Error::InvalidPart | Error::InvalidPartOrder | Error::EntityTooSmall + | Error::InvalidDigest(_) + | Error::InvalidEncryptionAlgorithm(_) | Error::InvalidXml(_) | Error::InvalidUtf8Str(_) - | Error::InvalidUtf8String(_) - | Error::InvalidHeader(_) => StatusCode::BAD_REQUEST, + | Error::InvalidUtf8String(_) => StatusCode::BAD_REQUEST, } } @@ -189,22 +197,23 @@ impl ApiError for Error { } } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = s3_xml::Error { code: s3_xml::Value(self.aws_code().to_string()), message: s3_xml::Value(format!("{}", self)), resource: Some(s3_xml::Value(path.to_string())), region: Some(s3_xml::Value(garage_region.to_string())), }; - Body::from(s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { + let error_str = s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { r#" - InternalError - XML encoding of error failed + InternalError + XML encoding of error failed - "# + "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 5e682726..a1e4ce10 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -1,32 +1,57 @@ //! Function related to GET and HEAD requests +use std::collections::BTreeMap; +use std::convert::TryInto; use std::sync::Arc; -use std::time::{Duration, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use bytes::Bytes; use futures::future; -use futures::stream::{self, StreamExt}; +use futures::stream::{self, Stream, StreamExt}; use http::header::{ - ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, - IF_NONE_MATCH, LAST_MODIFIED, RANGE, + HeaderMap, HeaderName, ACCEPT_RANGES, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, + CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, EXPIRES, IF_MATCH, + IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE, LAST_MODIFIED, RANGE, }; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use tokio::sync::mpsc; -use garage_rpc::rpc_helper::{netapp::stream::ByteStream, OrderTag}; +use garage_net::stream::ByteStream; +use garage_rpc::rpc_helper::OrderTag; use garage_table::EmptyKey; use garage_util::data::*; -use garage_util::error::OkOrMessage; +use garage_util::error::{Error as UtilError, OkOrMessage}; use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::s3::error::*; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; -const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; +use crate::api_server::ResBody; +use crate::copy::*; +use crate::encryption::EncryptionParams; +use crate::error::*; + +const X_AMZ_MP_PARTS_COUNT: HeaderName = HeaderName::from_static("x-amz-mp-parts-count"); + +#[derive(Default)] +pub struct GetObjectOverrides { + pub(crate) response_cache_control: Option, + pub(crate) response_content_disposition: Option, + pub(crate) response_content_encoding: Option, + pub(crate) response_content_language: Option, + pub(crate) response_content_type: Option, + pub(crate) response_expires: Option, +} fn object_headers( version: &ObjectVersion, version_meta: &ObjectVersionMeta, + meta_inner: &ObjectVersionMetaInner, + encryption: EncryptionParams, + checksum_mode: ChecksumMode, ) -> http::response::Builder { debug!("Version meta: {:?}", version_meta); @@ -34,7 +59,6 @@ fn object_headers( let date_str = httpdate::fmt_http_date(date); let mut resp = Response::builder() - .header(CONTENT_TYPE, version_meta.headers.content_type.to_string()) .header(LAST_MODIFIED, date_str) .header(ACCEPT_RANGES, "bytes".to_string()); @@ -42,60 +66,94 @@ fn object_headers( resp = resp.header(ETAG, format!("\"{}\"", version_meta.etag)); } - for (k, v) in version_meta.headers.other.iter() { - resp = resp.header(k, v.to_string()); + // When metadata is retrieved through the REST API, Amazon S3 combines headers that + // have the same name (ignoring case) into a comma-delimited list. + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html + let mut headers_by_name = BTreeMap::new(); + for (name, value) in meta_inner.headers.iter() { + let name_lower = name.to_ascii_lowercase(); + headers_by_name + .entry(name_lower) + .or_insert(vec![]) + .push(value.as_str()); } + for (name, values) in headers_by_name { + resp = resp.header(name, values.join(",")); + } + + if checksum_mode.enabled { + resp = add_checksum_response_headers(&meta_inner.checksum, resp); + } + + encryption.add_response_headers(&mut resp); + resp } -fn try_answer_cached( +/// Override headers according to specific query parameters, see +/// section "Overriding response header values through the request" in +/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html +fn getobject_override_headers( + overrides: GetObjectOverrides, + resp: &mut http::response::Builder, +) -> Result<(), Error> { + // TODO: this only applies for signed requests, so when we support + // anonymous access in the future we will have to do a permission check here + let overrides = [ + (CACHE_CONTROL, overrides.response_cache_control), + (CONTENT_DISPOSITION, overrides.response_content_disposition), + (CONTENT_ENCODING, overrides.response_content_encoding), + (CONTENT_LANGUAGE, overrides.response_content_language), + (CONTENT_TYPE, overrides.response_content_type), + (EXPIRES, overrides.response_expires), + ]; + for (hdr, val_opt) in overrides { + if let Some(val) = val_opt { + let val = val.try_into().ok_or_bad_request("invalid header value")?; + resp.headers_mut().unwrap().insert(hdr, val); + } + } + Ok(()) +} + +fn handle_http_precondition( version: &ObjectVersion, version_meta: &ObjectVersionMeta, - req: &Request, -) -> Option> { - // It is possible, and is even usually the case, [that both If-None-Match and - // If-Modified-Since] are present in a request. In this situation If-None-Match takes - // precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational - // being that etag based matching is more accurate, it has no issue with sub-second precision - // for instance (in case of very fast updates) - let cached = if let Some(none_match) = req.headers().get(IF_NONE_MATCH) { - let none_match = none_match.to_str().ok()?; - let expected = format!("\"{}\"", version_meta.etag); - let found = none_match - .split(',') - .map(str::trim) - .any(|etag| etag == expected || etag == "\"*\""); - found - } else if let Some(modified_since) = req.headers().get(IF_MODIFIED_SINCE) { - let modified_since = modified_since.to_str().ok()?; - let client_date = httpdate::parse_http_date(modified_since).ok()?; - let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp); - client_date >= server_date - } else { - false - }; + req: &Request<()>, +) -> Result>, Error> { + let precondition_headers = PreconditionHeaders::parse(req)?; - if cached { - Some( + if let Some(status_code) = precondition_headers.check(&version, &version_meta.etag)? { + Ok(Some( Response::builder() - .status(StatusCode::NOT_MODIFIED) - .body(Body::empty()) + .status(status_code) + .body(empty_body()) .unwrap(), - ) + )) } else { - None + Ok(None) } } /// Handle HEAD request pub async fn handle_head( + ctx: ReqCtx, + req: &Request<()>, + key: &str, + part_number: Option, +) -> Result, Error> { + handle_head_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number).await +} + +/// Handle HEAD request for website +pub async fn handle_head_without_ctx( garage: Arc, - req: &Request, + req: &Request<()>, bucket_id: Uuid, key: &str, part_number: Option, -) -> Result, Error> { +) -> Result, Error> { let object = garage .object_table .get(&bucket_id, &key.to_string()) @@ -120,25 +178,37 @@ pub async fn handle_head( _ => unreachable!(), }; - if let Some(cached) = try_answer_cached(object_version, version_meta, req) { - return Ok(cached); + if let Some(res) = handle_http_precondition(object_version, version_meta, req)? { + return Ok(res); } + let (encryption, headers) = + EncryptionParams::check_decrypt(&garage, req.headers(), &version_meta.encryption)?; + + let checksum_mode = checksum_mode(&req); + if let Some(pn) = part_number { match version_data { - ObjectVersionData::Inline(_, bytes) => { + ObjectVersionData::Inline(_, _) => { if pn != 1 { return Err(Error::InvalidPart); } - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", bytes.len())) - .header( - CONTENT_RANGE, - format!("bytes 0-{}/{}", bytes.len() - 1, bytes.len()), - ) - .header(X_AMZ_MP_PARTS_COUNT, "1") - .status(StatusCode::PARTIAL_CONTENT) - .body(Body::empty())?) + let bytes_len = version_meta.size; + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", bytes_len)) + .header( + CONTENT_RANGE, + format!("bytes 0-{}/{}", bytes_len - 1, bytes_len), + ) + .header(X_AMZ_MP_PARTS_COUNT, "1") + .status(StatusCode::PARTIAL_CONTENT) + .body(empty_body())?) } ObjectVersionData::FirstBlock(_, _) => { let version = garage @@ -146,43 +216,68 @@ pub async fn handle_head( .get(&object_version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; let (part_offset, part_end) = calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?; - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", part_end - part_offset)) - .header( - CONTENT_RANGE, - format!( - "bytes {}-{}/{}", - part_offset, - part_end - 1, - version_meta.size - ), - ) - .header(X_AMZ_MP_PARTS_COUNT, format!("{}", version.n_parts()?)) - .status(StatusCode::PARTIAL_CONTENT) - .body(Body::empty())?) + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", part_end - part_offset)) + .header( + CONTENT_RANGE, + format!( + "bytes {}-{}/{}", + part_offset, + part_end - 1, + version_meta.size + ), + ) + .header(X_AMZ_MP_PARTS_COUNT, format!("{}", version.n_parts()?)) + .status(StatusCode::PARTIAL_CONTENT) + .body(empty_body())?) } _ => unreachable!(), } } else { - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", version_meta.size)) - .status(StatusCode::OK) - .body(Body::empty())?) + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", version_meta.size)) + .status(StatusCode::OK) + .body(empty_body())?) } } /// Handle GET request pub async fn handle_get( + ctx: ReqCtx, + req: &Request<()>, + key: &str, + part_number: Option, + overrides: GetObjectOverrides, +) -> Result, Error> { + handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await +} + +/// Handle GET request +pub async fn handle_get_without_ctx( garage: Arc, - req: &Request, + req: &Request<()>, bucket_id: Uuid, key: &str, part_number: Option, -) -> Result, Error> { + overrides: GetObjectOverrides, +) -> Result, Error> { let object = garage .object_table .get(&bucket_id, &key.to_string()) @@ -206,49 +301,136 @@ pub async fn handle_get( ObjectVersionData::FirstBlock(meta, _) => meta, }; - if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) { - return Ok(cached); + if let Some(res) = handle_http_precondition(last_v, last_v_meta, req)? { + return Ok(res); } + let (enc, headers) = + EncryptionParams::check_decrypt(&garage, req.headers(), &last_v_meta.encryption)?; + + let checksum_mode = checksum_mode(&req); + match (part_number, parse_range_header(req, last_v_meta.size)?) { - (Some(_), Some(_)) => { - return Err(Error::bad_request( - "Cannot specify both partNumber and Range header", - )); - } + (Some(_), Some(_)) => Err(Error::bad_request( + "Cannot specify both partNumber and Range header", + )), (Some(pn), None) => { - return handle_get_part(garage, last_v, last_v_data, last_v_meta, pn).await; - } - (None, Some(range)) => { - return handle_get_range( + handle_get_part( garage, last_v, last_v_data, last_v_meta, + enc, + &headers, + pn, + ChecksumMode { + // TODO: for multipart uploads, checksums of each part should be stored + // so that we can return the corresponding checksum here + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, + ) + .await + } + (None, Some(range)) => { + handle_get_range( + garage, + last_v, + last_v_data, + last_v_meta, + enc, + &headers, range.start, range.start + range.length, + ChecksumMode { + // TODO: for range queries that align with part boundaries, + // we should return the saved checksum of the part + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, ) - .await; + .await + } + (None, None) => { + handle_get_full( + garage, + last_v, + last_v_data, + last_v_meta, + enc, + &headers, + overrides, + checksum_mode, + ) + .await } - (None, None) => (), } +} - let resp_builder = object_headers(last_v, last_v_meta) - .header(CONTENT_LENGTH, format!("{}", last_v_meta.size)) - .status(StatusCode::OK); +pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> { + if version.deleted.get() { + // the version was deleted between when the object_table was consulted + // and now, this could mean the object was deleted, or overriden. + // Rather than say the key doesn't exist, return a transient error + // to signal the client to try again. + return Err(CommonError::InternalError(UtilError::Message( + "conflict/inconsistency between object and version state, version is deleted" + .to_string(), + )) + .into()); + } + Ok(()) +} - match &last_v_data { +async fn handle_get_full( + garage: Arc, + version: &ObjectVersion, + version_data: &ObjectVersionData, + version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, + overrides: GetObjectOverrides, + checksum_mode: ChecksumMode, +) -> Result, Error> { + let mut resp_builder = object_headers( + version, + version_meta, + &meta_inner, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", version_meta.size)) + .status(StatusCode::OK); + getobject_override_headers(overrides, &mut resp_builder)?; + + let stream = full_object_byte_stream(garage, version, version_data, encryption); + + Ok(resp_builder.body(response_body_from_stream(stream))?) +} + +pub fn full_object_byte_stream( + garage: Arc, + version: &ObjectVersion, + version_data: &ObjectVersionData, + encryption: EncryptionParams, +) -> ByteStream { + match &version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_, bytes) => { - let body: Body = Body::from(bytes.to_vec()); - Ok(resp_builder.body(body)?) + let bytes = bytes.to_vec(); + Box::pin(futures::stream::once(async move { + encryption + .decrypt_blob(&bytes) + .map(|x| Bytes::from(x.to_vec())) + .map_err(std_error_from_read_error) + })) } ObjectVersionData::FirstBlock(_, first_block_hash) => { - let (tx, rx) = mpsc::channel(2); + let (tx, rx) = mpsc::channel::(2); let order_stream = OrderTag::stream(); let first_block_hash = *first_block_hash; - let version_uuid = last_v.uuid; + let version_uuid = version.uuid; tokio::spawn(async move { match async { @@ -257,19 +439,19 @@ pub async fn handle_get( garage2.version_table.get(&version_uuid, &EmptyKey).await }); - let stream_block_0 = garage - .block_manager - .rpc_get_block_streaming(&first_block_hash, Some(order_stream.order(0))) + let stream_block_0 = encryption + .get_block(&garage, &first_block_hash, Some(order_stream.order(0))) .await?; + tx.send(stream_block_0) .await .ok_or_message("channel closed")?; let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) { - let stream_block_i = garage - .block_manager - .rpc_get_block_streaming(&vb.hash, Some(order_stream.order(i as u64))) + let stream_block_i = encryption + .get_block(&garage, &vb.hash, Some(order_stream.order(i as u64))) .await?; tx.send(stream_block_i) .await @@ -282,21 +464,20 @@ pub async fn handle_get( { Ok(()) => (), Err(e) => { - let err = std::io::Error::new( - std::io::ErrorKind::Other, - format!("Error while getting object data: {}", e), - ); - let _ = tx - .send(Box::pin(stream::once(future::ready(Err(err))))) - .await; + // TODO i think this is a bad idea, we should log + // an error and stop there. If the error happens to + // be exactly the size of what hasn't been streamed + // yet, the client will see the request as a + // success + // instead truncating the output notify the client + // something happened with their download, so that + // they can retry it + let _ = tx.send(error_stream_item(e)).await; } } }); - let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten(); - - let body = hyper::body::Body::wrap_stream(body_stream); - Ok(resp_builder.body(body)?) + Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx).flatten()) } } } @@ -306,10 +487,16 @@ async fn handle_get_range( version: &ObjectVersion, version_data: &ObjectVersionData, version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, begin: u64, end: u64, -) -> Result, Error> { - let resp_builder = object_headers(version, version_meta) + checksum_mode: ChecksumMode, +) -> Result, Error> { + // Here we do not use getobject_override_headers because we don't + // want to add any overridden headers (those should not be added + // when returning PARTIAL_CONTENT) + let resp_builder = object_headers(version, version_meta, meta_inner, encryption, checksum_mode) .header(CONTENT_LENGTH, format!("{}", end - begin)) .header( CONTENT_RANGE, @@ -320,8 +507,9 @@ async fn handle_get_range( match &version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_meta, bytes) => { + let bytes = encryption.decrypt_blob(&bytes)?; if end as usize <= bytes.len() { - let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec()); + let body = bytes_body(bytes[begin as usize..end as usize].to_vec().into()); Ok(resp_builder.body(body)?) } else { Err(Error::internal_error( @@ -335,8 +523,9 @@ async fn handle_get_range( .get(&version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; - - let body = body_from_blocks_range(garage, version.blocks.items(), begin, end); + check_version_not_deleted(&version)?; + let body = + body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); Ok(resp_builder.body(body)?) } } @@ -347,16 +536,28 @@ async fn handle_get_part( object_version: &ObjectVersion, version_data: &ObjectVersionData, version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, part_number: u64, -) -> Result, Error> { - let resp_builder = - object_headers(object_version, version_meta).status(StatusCode::PARTIAL_CONTENT); + checksum_mode: ChecksumMode, +) -> Result, Error> { + // Same as for get_range, no getobject_override_headers + let resp_builder = object_headers( + object_version, + version_meta, + meta_inner, + encryption, + checksum_mode, + ) + .status(StatusCode::PARTIAL_CONTENT); match version_data { ObjectVersionData::Inline(_, bytes) => { if part_number != 1 { return Err(Error::InvalidPart); } + let bytes = encryption.decrypt_blob(&bytes)?; + assert_eq!(bytes.len() as u64, version_meta.size); Ok(resp_builder .header(CONTENT_LENGTH, format!("{}", bytes.len())) .header( @@ -364,7 +565,7 @@ async fn handle_get_part( format!("bytes {}-{}/{}", 0, bytes.len() - 1, bytes.len()), ) .header(X_AMZ_MP_PARTS_COUNT, "1") - .body(Body::from(bytes.to_vec()))?) + .body(bytes_body(bytes.into_owned().into()))?) } ObjectVersionData::FirstBlock(_, _) => { let version = garage @@ -373,10 +574,13 @@ async fn handle_get_part( .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; + let (begin, end) = calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?; - let body = body_from_blocks_range(garage, version.blocks.items(), begin, end); + let body = + body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); Ok(resp_builder .header(CONTENT_LENGTH, format!("{}", end - begin)) @@ -392,7 +596,7 @@ async fn handle_get_part( } fn parse_range_header( - req: &Request, + req: &Request<()>, total_size: u64, ) -> Result, Error> { let range = match req.headers().get(RANGE) { @@ -429,12 +633,27 @@ fn calculate_part_bounds(v: &Version, part_number: u64) -> Option<(u64, u64)> { None } +struct ChecksumMode { + enabled: bool, +} + +fn checksum_mode(req: &Request<()>) -> ChecksumMode { + ChecksumMode { + enabled: req + .headers() + .get(X_AMZ_CHECKSUM_MODE) + .map(|x| x == "ENABLED") + .unwrap_or(false), + } +} + fn body_from_blocks_range( garage: Arc, + encryption: EncryptionParams, all_blocks: &[(VersionBlockKey, VersionBlock)], begin: u64, end: u64, -) -> Body { +) -> ResBody { // We will store here the list of blocks that have an intersection with the requested // range, as well as their "true offset", which is their actual offset in the complete // file (whereas block.offset designates the offset of the block WITHIN THE PART @@ -456,17 +675,16 @@ fn body_from_blocks_range( } let order_stream = OrderTag::stream(); - let body_stream = futures::stream::iter(blocks) - .enumerate() - .map(move |(i, (block, block_offset))| { - let garage = garage.clone(); - async move { - garage - .block_manager - .rpc_get_block_streaming(&block.hash, Some(order_stream.order(i as u64))) - .await - .unwrap_or_else(|e| error_stream(i, e)) - .scan(block_offset, move |chunk_offset, chunk| { + let (tx, rx) = mpsc::channel::(2); + + tokio::spawn(async move { + match async { + for (i, (block, block_offset)) in blocks.iter().enumerate() { + let block_stream = encryption + .get_block(&garage, &block.hash, Some(order_stream.order(i as u64))) + .await?; + let block_stream = block_stream + .scan(*block_offset, move |chunk_offset, chunk| { let r = match chunk { Ok(chunk_bytes) => { let chunk_len = chunk_bytes.len() as u64; @@ -502,20 +720,168 @@ fn body_from_blocks_range( }; futures::future::ready(r) }) - .filter_map(futures::future::ready) + .filter_map(futures::future::ready); + + let block_stream: ByteStream = Box::pin(block_stream); + tx.send(Box::pin(block_stream)) + .await + .ok_or_message("channel closed")?; } + + Ok::<(), Error>(()) + } + .await + { + Ok(()) => (), + Err(e) => { + let _ = tx.send(error_stream_item(e)).await; + } + } + }); + + response_body_from_block_stream(rx) +} + +fn response_body_from_block_stream(rx: mpsc::Receiver) -> ResBody { + let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten(); + response_body_from_stream(body_stream) +} + +fn response_body_from_stream(stream: S) -> ResBody +where + S: Stream> + Send + Sync + 'static, +{ + let body_stream = stream.map(|x| { + x.map(hyper::body::Frame::data) + .map_err(|e| Error::from(garage_util::error::Error::from(e))) + }); + ResBody::new(http_body_util::StreamBody::new(body_stream)) +} + +fn error_stream_item(e: E) -> ByteStream { + Box::pin(stream::once(future::ready(Err(std_error_from_read_error( + e, + ))))) +} + +fn std_error_from_read_error(e: E) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Error while reading object data: {}", e), + ) +} + +// ---- + +pub struct PreconditionHeaders { + if_match: Option>, + if_modified_since: Option, + if_none_match: Option>, + if_unmodified_since: Option, +} + +impl PreconditionHeaders { + fn parse(req: &Request) -> Result { + Self::parse_with( + req.headers(), + &IF_MATCH, + &IF_NONE_MATCH, + &IF_MODIFIED_SINCE, + &IF_UNMODIFIED_SINCE, + ) + } + + pub(crate) fn parse_copy_source(req: &Request) -> Result { + Self::parse_with( + req.headers(), + &X_AMZ_COPY_SOURCE_IF_MATCH, + &X_AMZ_COPY_SOURCE_IF_NONE_MATCH, + &X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE, + &X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE, + ) + } + + fn parse_with( + headers: &HeaderMap, + hdr_if_match: &HeaderName, + hdr_if_none_match: &HeaderName, + hdr_if_modified_since: &HeaderName, + hdr_if_unmodified_since: &HeaderName, + ) -> Result { + Ok(Self { + if_match: headers + .get(hdr_if_match) + .map(|x| x.to_str()) + .transpose()? + .map(|x| { + x.split(',') + .map(|m| m.trim().trim_matches('"').to_string()) + .collect::>() + }), + if_none_match: headers + .get(hdr_if_none_match) + .map(|x| x.to_str()) + .transpose()? + .map(|x| { + x.split(',') + .map(|m| m.trim().trim_matches('"').to_string()) + .collect::>() + }), + if_modified_since: headers + .get(hdr_if_modified_since) + .map(|x| x.to_str()) + .transpose()? + .map(httpdate::parse_http_date) + .transpose() + .ok_or_bad_request("Invalid date in if-modified-since")?, + if_unmodified_since: headers + .get(hdr_if_unmodified_since) + .map(|x| x.to_str()) + .transpose()? + .map(httpdate::parse_http_date) + .transpose() + .ok_or_bad_request("Invalid date in if-unmodified-since")?, }) - .buffered(2) - .flatten(); + } - hyper::body::Body::wrap_stream(body_stream) -} + fn check(&self, v: &ObjectVersion, etag: &str) -> Result, Error> { + // we store date with ms precision, but headers are precise to the second: truncate + // the timestamp to handle the same-second edge case + let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000); -fn error_stream(i: usize, e: garage_util::error::Error) -> ByteStream { - Box::pin(futures::stream::once(async move { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("Could not get block {}: {}", i, e), - )) - })) + // Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6 + + if let Some(im) = &self.if_match { + // Step 1: if-match is present + if !im.iter().any(|x| x == etag || x == "*") { + return Ok(Some(StatusCode::PRECONDITION_FAILED)); + } + } else if let Some(ius) = &self.if_unmodified_since { + // Step 2: if-unmodified-since is present, and if-match is absent + if v_date > *ius { + return Ok(Some(StatusCode::PRECONDITION_FAILED)); + } + } + + if let Some(inm) = &self.if_none_match { + // Step 3: if-none-match is present + if inm.iter().any(|x| x == etag || x == "*") { + return Ok(Some(StatusCode::NOT_MODIFIED)); + } + } else if let Some(ims) = &self.if_modified_since { + // Step 4: if-modified-since is present, and if-none-match is absent + if v_date <= *ims { + return Ok(Some(StatusCode::NOT_MODIFIED)); + } + } + + Ok(None) + } + + pub(crate) fn check_copy_source(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> { + match self.check(v, etag)? { + Some(_) => Err(Error::PreconditionFailed), + None => Ok(()), + } + } } diff --git a/src/api/s3/mod.rs b/src/api/s3/lib.rs similarity index 73% rename from src/api/s3/mod.rs rename to src/api/s3/lib.rs index cbdb94ab..83f684f8 100644 --- a/src/api/s3/mod.rs +++ b/src/api/s3/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; pub mod error; @@ -11,7 +14,8 @@ mod list; mod multipart; mod post_object; mod put; -mod website; +pub mod website; +mod encryption; mod router; pub mod xml; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index 1e7d6755..ccda6cfd 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -1,83 +1,82 @@ use quick_xml::de::from_reader; -use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule, }; -use garage_model::garage::Garage; -use garage_util::data::*; -pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; - if let Some(lifecycle) = param.lifecycle_config.get() { + if let Some(lifecycle) = bucket_params.lifecycle_config.get() { let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle); let xml = to_xml_with_header(&wc)?; Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } else { Ok(Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .status(StatusCode::NOT_FOUND) + .body(empty_body())?) } } -pub async fn handle_delete_lifecycle( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.lifecycle_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.lifecycle_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_put_lifecycle( - garage: Arc, - mut bucket: Bucket, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let body = req.into_body().collect().await?; let conf: LifecycleConfiguration = from_reader(&body as &[u8])?; let config = conf .validate_into_garage_lifecycle_config() .ok_or_bad_request("Invalid lifecycle configuration")?; - param.lifecycle_config.update(Some(config)); - garage.bucket_table.insert(&bucket).await?; + bucket_params.lifecycle_config.update(Some(config)); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) - .body(Body::empty())?) + .body(empty_body())?) } // ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index 33d62518..94c2c895 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -1,25 +1,26 @@ use std::collections::{BTreeMap, BTreeSet}; use std::iter::{Iterator, Peekable}; -use std::sync::Arc; use base64::prelude::*; -use hyper::{Body, Response}; +use hyper::{Request, Response}; use garage_util::data::*; use garage_util::error::Error as GarageError; use garage_util::time::*; -use garage_model::garage::Garage; use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_table::EnumerationOrder; -use crate::encoding::*; -use crate::helpers::key_after_prefix; -use crate::s3::error::*; -use crate::s3::multipart as s3_multipart; -use crate::s3::xml as s3_xml; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::multipart as s3_multipart; +use crate::xml as s3_xml; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; @@ -53,7 +54,6 @@ pub struct ListMultipartUploadsQuery { #[derive(Debug)] pub struct ListPartsQuery { pub bucket_name: String, - pub bucket_id: Uuid, pub key: String, pub upload_id: String, pub part_number_marker: Option, @@ -61,9 +61,10 @@ pub struct ListPartsQuery { } pub async fn handle_list( - garage: Arc, + ctx: ReqCtx, query: &ListObjectsQuery, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; let io = |bucket, key, count| { let t = &garage.object_table; async move { @@ -162,13 +163,15 @@ pub async fn handle_list( let xml = s3_xml::to_xml_with_header(&result)?; Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } pub async fn handle_list_multipart_upload( - garage: Arc, + ctx: ReqCtx, query: &ListMultipartUploadsQuery, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + let io = |bucket, key, count| { let t = &garage.object_table; async move { @@ -264,19 +267,26 @@ pub async fn handle_list_multipart_upload( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } pub async fn handle_list_parts( - garage: Arc, + ctx: ReqCtx, + req: Request, query: &ListPartsQuery, -) -> Result, Error> { +) -> Result, Error> { debug!("ListParts {:?}", query); let upload_id = s3_multipart::decode_upload_id(&query.upload_id)?; - let (_, _, mpu) = - s3_multipart::get_upload(&garage, &query.bucket_id, &query.key, &upload_id).await?; + let (_, object_version, mpu) = s3_multipart::get_upload(&ctx, &query.key, &upload_id).await?; + + let object_encryption = match object_version.state { + ObjectVersionState::Uploading { encryption, .. } => encryption, + _ => unreachable!(), + }; + let encryption_res = + EncryptionParams::check_decrypt(&ctx.garage, req.headers(), &object_encryption); let (info, next) = fetch_part_info(query, &mpu)?; @@ -295,11 +305,40 @@ pub async fn handle_list_parts( is_truncated: s3_xml::Value(format!("{}", next.is_some())), parts: info .iter() - .map(|part| s3_xml::PartItem { - etag: s3_xml::Value(format!("\"{}\"", part.etag)), - last_modified: s3_xml::Value(msec_to_rfc3339(part.timestamp)), - part_number: s3_xml::IntValue(part.part_number as i64), - size: s3_xml::IntValue(part.size as i64), + .map(|part| { + // hide checksum if object is encrypted and the decryption + // keys are not provided + let checksum = part.checksum.filter(|_| encryption_res.is_ok()); + s3_xml::PartItem { + etag: s3_xml::Value(format!("\"{}\"", part.etag)), + last_modified: s3_xml::Value(msec_to_rfc3339(part.timestamp)), + part_number: s3_xml::IntValue(part.part_number as i64), + size: s3_xml::IntValue(part.size as i64), + checksum_crc32: match &checksum { + Some(ChecksumValue::Crc32(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_crc32c: match &checksum { + Some(ChecksumValue::Crc32c(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_sha1: match &checksum { + Some(ChecksumValue::Sha1(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_sha256: match &checksum { + Some(ChecksumValue::Sha256(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + } }) .collect(), @@ -319,7 +358,7 @@ pub async fn handle_list_parts( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } /* @@ -345,6 +384,7 @@ struct PartInfo<'a> { timestamp: u64, part_number: u64, size: u64, + checksum: Option, } enum ExtractionResult { @@ -358,7 +398,7 @@ enum ExtractionResult { key: String, }, // Fallback key is used for legacy APIs that only support - // exlusive pagination (and not inclusive one). + // exclusive pagination (and not inclusive one). SkipTo { key: String, fallback_key: Option, @@ -368,7 +408,7 @@ enum ExtractionResult { #[derive(PartialEq, Clone, Debug)] enum RangeBegin { // Fallback key is used for legacy APIs that only support - // exlusive pagination (and not inclusive one). + // exclusive pagination (and not inclusive one). IncludingKey { key: String, fallback_key: Option, @@ -426,8 +466,10 @@ where // Drop the first key if needed // Only AfterKey requires it according to the S3 spec and our implem. match (&cursor, iter.peek()) { - (RangeBegin::AfterKey { key }, Some(object)) if &object.key == key => iter.next(), - (_, _) => None, + (RangeBegin::AfterKey { key }, Some(object)) if &object.key == key => { + iter.next(); + } + _ => (), }; while let Some(object) = iter.peek() { @@ -436,16 +478,22 @@ where return Ok(None); } - cursor = match acc.extract(query, &cursor, &mut iter) { - ExtractionResult::Extracted { key } => RangeBegin::AfterKey { key }, + match acc.extract(query, &cursor, &mut iter) { + ExtractionResult::Extracted { key } => { + cursor = RangeBegin::AfterKey { key }; + } ExtractionResult::SkipTo { key, fallback_key } => { - RangeBegin::IncludingKey { key, fallback_key } + cursor = RangeBegin::IncludingKey { key, fallback_key }; } ExtractionResult::FilledAtUpload { key, upload } => { - return Ok(Some(RangeBegin::AfterUpload { key, upload })) + return Ok(Some(RangeBegin::AfterUpload { key, upload })); + } + ExtractionResult::Filled => { + return Ok(Some(cursor)); + } + ExtractionResult::NoMore => { + return Ok(None); } - ExtractionResult::Filled => return Ok(Some(cursor)), - ExtractionResult::NoMore => return Ok(None), }; } @@ -477,6 +525,7 @@ fn fetch_part_info<'a>( timestamp: pk.timestamp, etag, size, + checksum: p.checksum, }; match parts.last_mut() { Some(lastpart) if lastpart.part_number == pk.part_number => { @@ -519,8 +568,8 @@ fn fetch_part_info<'a>( /// This key can be the prefix in the base case, or intermediate /// points in the dataset if we are continuing a previous listing. impl ListObjectsQuery { - fn build_accumulator(&self) -> Accumulator { - Accumulator::::new(self.common.page_size) + fn build_accumulator(&self) -> ObjectAccumulator { + ObjectAccumulator::new(self.common.page_size) } fn begin(&self) -> Result { @@ -529,9 +578,10 @@ impl ListObjectsQuery { // In V2 mode, the continuation token is defined as an opaque // string in the spec, so we can do whatever we want with it. // In our case, it is defined as either [ or ] (for include + // or exclude), followed by a base64-encoded string // representing the key to start with. - (Some(token), _) => match &token[..1] { - "[" => Ok(RangeBegin::IncludingKey { + (Some(token), _) => match &token.get(..1) { + Some("[") => Ok(RangeBegin::IncludingKey { key: String::from_utf8( BASE64_STANDARD .decode(token[1..].as_bytes()) @@ -539,7 +589,7 @@ impl ListObjectsQuery { )?, fallback_key: None, }), - "]" => Ok(RangeBegin::AfterKey { + Some("]") => Ok(RangeBegin::AfterKey { key: String::from_utf8( BASE64_STANDARD .decode(token[1..].as_bytes()) @@ -580,8 +630,8 @@ impl ListObjectsQuery { } impl ListMultipartUploadsQuery { - fn build_accumulator(&self) -> Accumulator { - Accumulator::::new(self.common.page_size) + fn build_accumulator(&self) -> UploadAccumulator { + UploadAccumulator::new(self.common.page_size) } fn begin(&self) -> Result { @@ -665,6 +715,7 @@ impl Accumulator { Some(p) => p, None => return None, }; + assert!(pfx.starts_with(&query.prefix)); // Try to register this prefix // If not possible, we can return early @@ -675,8 +726,11 @@ impl Accumulator { // We consume the whole common prefix from the iterator let mut last_pfx_key = &object.key; loop { - last_pfx_key = match objects.peek() { - Some(o) if o.key.starts_with(pfx) => &o.key, + match objects.peek() { + Some(o) if o.key.starts_with(pfx) => { + last_pfx_key = &o.key; + objects.next(); + } Some(_) => { return Some(ExtractionResult::Extracted { key: last_pfx_key.to_owned(), @@ -692,8 +746,6 @@ impl Accumulator { } } }; - - objects.next(); } } @@ -708,12 +760,11 @@ impl Accumulator { } // Otherwise, we need to check if we can add it - match self.is_full() { - true => false, - false => { - self.common_prefixes.insert(key); - true - } + if self.is_full() { + false + } else { + self.common_prefixes.insert(key); + true } } @@ -721,12 +772,11 @@ impl Accumulator { // It is impossible to add twice a key, this is an error assert!(!self.keys.contains_key(&key)); - match self.is_full() { - true => false, - false => { - self.keys.insert(key, value); - true - } + if self.is_full() { + false + } else { + self.keys.insert(key, value); + true } } } @@ -743,6 +793,7 @@ impl ExtractAccumulator for ObjectAccumulator { } let object = objects.next().expect("This iterator can not be empty as it is checked earlier in the code. This is a logic bug, please report it."); + assert!(object.key.starts_with(&query.prefix)); let version = match object.versions().iter().find(|x| x.is_data()) { Some(v) => v, @@ -933,10 +984,13 @@ mod tests { timestamp: TS, state: ObjectVersionState::Uploading { multipart: true, - headers: ObjectVersionHeaders { - content_type: "text/plain".to_string(), - other: BTreeMap::::new(), + encryption: ObjectVersionEncryption::Plaintext { + inner: ObjectVersionMetaInner { + headers: vec![], + checksum: None, + }, }, + checksum_algorithm: None, }, } } @@ -1125,6 +1179,7 @@ mod tests { version: uuid, size: Some(3), etag: Some("etag1".into()), + checksum: None, }, ), ( @@ -1136,6 +1191,7 @@ mod tests { version: uuid, size: None, etag: None, + checksum: None, }, ), ( @@ -1147,6 +1203,7 @@ mod tests { version: uuid, size: Some(10), etag: Some("etag2".into()), + checksum: None, }, ), ( @@ -1158,6 +1215,7 @@ mod tests { version: uuid, size: Some(7), etag: Some("etag3".into()), + checksum: None, }, ), ( @@ -1169,6 +1227,7 @@ mod tests { version: uuid, size: Some(5), etag: Some("etag4".into()), + checksum: None, }, ), ]; @@ -1185,10 +1244,8 @@ mod tests { #[test] fn test_fetch_part_info() -> Result<(), Error> { - let uuid = Uuid::from([0x08; 32]); let mut query = ListPartsQuery { bucket_name: "a".to_string(), - bucket_id: uuid, key: "a".to_string(), upload_id: "xx".to_string(), part_number_marker: None, @@ -1207,12 +1264,14 @@ mod tests { etag: "etag1", timestamp: TS, part_number: 1, - size: 3 + size: 3, + checksum: None, }, PartInfo { etag: "etag2", timestamp: TS, part_number: 3, + checksum: None, size: 10 }, ] @@ -1228,12 +1287,14 @@ mod tests { PartInfo { etag: "etag3", timestamp: TS, + checksum: None, part_number: 5, size: 7 }, PartInfo { etag: "etag4", timestamp: TS, + checksum: None, part_number: 8, size: 5 }, @@ -1257,24 +1318,28 @@ mod tests { PartInfo { etag: "etag1", timestamp: TS, + checksum: None, part_number: 1, size: 3 }, PartInfo { etag: "etag2", timestamp: TS, + checksum: None, part_number: 3, size: 10 }, PartInfo { etag: "etag3", timestamp: TS, + checksum: None, part_number: 5, size: 7 }, PartInfo { etag: "etag4", timestamp: TS, + checksum: None, part_number: 8, size: 5 }, diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 52ea8e78..d6eb26cb 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -1,40 +1,65 @@ use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; +use std::hash::Hasher; use std::sync::Arc; +use base64::prelude::*; +use crc32c::Crc32cHasher as Crc32c; +use crc32fast::Hasher as Crc32; use futures::prelude::*; -use hyper::body::Body; use hyper::{Request, Response}; -use md5::{Digest as Md5Digest, Md5}; +use md5::{Digest, Md5}; +use sha1::Sha1; +use sha2::Sha256; use garage_table::*; -use garage_util::async_hash::*; use garage_util::data::*; -use garage_util::time::*; +use garage_util::error::OkOrMessage; -use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::s3::block_ref_table::*; use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::s3::error::*; -use crate::s3::put::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::*; +use crate::xml as s3_xml; // ---- pub async fn handle_create_multipart_upload( - garage: Arc, - req: &Request, - bucket_name: &str, - bucket_id: Uuid, - key: &str, -) -> Result, Error> { + ctx: ReqCtx, + req: &Request, + key: &String, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; + let existing_object = garage.object_table.get(&bucket_id, &key).await?; + let upload_id = gen_uuid(); - let timestamp = now_msec(); - let headers = get_headers(req.headers())?; + let timestamp = next_timestamp(existing_object.as_ref()); + + let headers = extract_metadata_headers(req.headers())?; + let meta = ObjectVersionMetaInner { + headers, + checksum: None, + }; + + // Determine whether object should be encrypted, and if so the key + let encryption = EncryptionParams::new_from_headers(&garage, req.headers())?; + let object_encryption = encryption.encrypt_meta(meta)?; + + let checksum_algorithm = request_checksum_algorithm(req.headers())?; // Create object in object table let object_version = ObjectVersion { @@ -42,16 +67,17 @@ pub async fn handle_create_multipart_upload( timestamp, state: ObjectVersionState::Uploading { multipart: true, - headers, + encryption: object_encryption, + checksum_algorithm, }, }; - let object = Object::new(bucket_id, key.to_string(), vec![object_version]); + let object = Object::new(*bucket_id, key.to_string(), vec![object_version]); garage.object_table.insert(&object).await?; // Create multipart upload in mpu table // This multipart upload will hold references to uploaded parts // (which are entries in the Version table) - let mpu = MultipartUpload::new(upload_id, timestamp, bucket_id, key.into(), false); + let mpu = MultipartUpload::new(upload_id, timestamp, *bucket_id, key.into(), false); garage.mpu_table.insert(&mpu).await?; // Send success response @@ -63,35 +89,65 @@ pub async fn handle_create_multipart_upload( }; let xml = s3_xml::to_xml_with_header(&result)?; - Ok(Response::new(Body::from(xml.into_bytes()))) + let mut resp = Response::builder(); + encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(xml))?) } pub async fn handle_put_part( - garage: Arc, - req: Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: Request, key: &str, part_number: u64, upload_id: &str, - content_sha256: Option, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + let upload_id = decode_upload_id(upload_id)?; - let content_md5 = match req.headers().get("content-md5") { - Some(x) => Some(x.to_str()?.to_string()), - None => None, + let expected_checksums = ExpectedChecksums { + md5: match req.headers().get("content-md5") { + Some(x) => Some(x.to_str()?.to_string()), + None => None, + }, + sha256: None, + extra: request_checksum_value(req.headers())?, }; - // Read first chuck, and at the same time try to get object to see if it exists let key = key.to_string(); - let body = req.into_body().map_err(Error::from); - let mut chunker = StreamChunker::new(body, garage.config.block_size); + let (req_head, mut req_body) = req.into_parts(); - let ((_, _, mut mpu), first_block) = futures::try_join!( - get_upload(&garage, &bucket_id, &key, &upload_id), - chunker.next(), - )?; + // Before we stream the body, configure the needed checksums. + req_body.add_expected_checksums(expected_checksums.clone()); + // TODO: avoid parsing encryption headers twice... + if !EncryptionParams::new_from_headers(&garage, &req_head.headers)?.is_encrypted() { + // For non-encrypted objects, we need to compute the md5sum in all cases + // (even if content-md5 is not set), because it is used as an etag of the + // part, which is in turn used in the etag computation of the whole object + req_body.add_md5(); + } + + let (stream, stream_checksums) = req_body.streaming_with_checksums(); + let stream = stream.map_err(Error::from); + + let mut chunker = StreamChunker::new(stream, garage.config.block_size); + + // Read first chuck, and at the same time try to get object to see if it exists + let ((_, object_version, mut mpu), first_block) = + futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?; + + // Check encryption params + let (object_encryption, checksum_algorithm) = match object_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), + _ => unreachable!(), + }; + let (encryption, _) = + EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?; // Check object is valid and part can be accepted let first_block = first_block.ok_or_bad_request("Empty body")?; @@ -118,7 +174,9 @@ pub async fn handle_put_part( mpu_part_key, MpuPart { version: version_uuid, + // all these are filled in later, at the end of this function etag: None, + checksum: None, size: None, }, ); @@ -132,33 +190,31 @@ pub async fn handle_put_part( garage.version_table.insert(&version).await?; // Copy data to version - let first_block_hash = async_blake2sum(first_block.clone()).await; - - let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks( - &garage, + let (total_size, _, _) = read_and_put_blocks( + &ctx, &version, + encryption, part_number, first_block, - first_block_hash, - &mut chunker, + chunker, + Checksummer::new(), ) .await?; - // Verify that checksums map - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; + // Verify that checksums match + let checksums = stream_checksums + .await + .ok_or_internal_error("checksum calculation")??; // Store part etag in version - let data_md5sum_hex = hex::encode(data_md5sum); + let etag = encryption.etag_from_md5(&checksums.md5); + mpu.parts.put( mpu_part_key, MpuPart { version: version_uuid, - etag: Some(data_md5sum_hex.clone()), + etag: Some(etag.clone()), + checksum: checksums.extract(checksum_algorithm), size: Some(total_size), }, ); @@ -168,11 +224,10 @@ pub async fn handle_put_part( // We won't have to clean up on drop. interrupted_cleanup.cancel(); - let response = Response::builder() - .header("ETag", format!("\"{}\"", data_md5sum_hex)) - .body(Body::empty()) - .unwrap(); - Ok(response) + let mut resp = Response::builder().header("ETag", format!("\"{}\"", etag)); + encryption.add_response_headers(&mut resp); + let resp = add_checksum_response_headers(&expected_checksums.extra, resp); + Ok(resp.body(empty_body())?) } struct InterruptedCleanup(Option); @@ -207,19 +262,22 @@ impl Drop for InterruptedCleanup { } pub async fn handle_complete_multipart_upload( - garage: Arc, - req: Request, - bucket_name: &str, - bucket: &Bucket, + ctx: ReqCtx, + req: Request, key: &str, upload_id: &str, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; + let (req_head, req_body) = req.into_parts(); - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } + let expected_checksum = request_checksum_value(&req_head.headers)?; + + let body = req_body.collect().await?; let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?; let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml) @@ -233,14 +291,18 @@ pub async fn handle_complete_multipart_upload( // Get object and multipart upload let key = key.to_string(); - let (_, mut object_version, mpu) = get_upload(&garage, &bucket.id, &key, &upload_id).await?; + let (object, mut object_version, mpu) = get_upload(&ctx, &key, &upload_id).await?; if mpu.parts.is_empty() { return Err(Error::bad_request("No data was uploaded")); } - let headers = match object_version.state { - ObjectVersionState::Uploading { headers, .. } => headers, + let (object_encryption, checksum_algorithm) = match object_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), _ => unreachable!(), }; @@ -268,6 +330,13 @@ pub async fn handle_complete_multipart_upload( for req_part in body_list_of_parts.iter() { match have_parts.get(&req_part.part_number) { Some(part) if part.etag.as_ref() == Some(&req_part.etag) && part.size.is_some() => { + // alternative version: if req_part.checksum.is_some() && part.checksum != req_part.checksum { + if part.checksum != req_part.checksum { + return Err(Error::InvalidDigest(format!( + "Invalid checksum for part {}: in request = {:?}, uploaded part = {:?}", + req_part.part_number, req_part.checksum, part.checksum + ))); + } parts.push(*part) } _ => return Err(Error::InvalidPart), @@ -287,7 +356,7 @@ pub async fn handle_complete_multipart_upload( let mut final_version = Version::new( upload_id, VersionBacklink::Object { - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.to_string(), }, false, @@ -315,83 +384,133 @@ pub async fn handle_complete_multipart_upload( }); garage.block_ref_table.insert_many(block_refs).await?; - // Calculate etag of final object + // Calculate checksum and etag of final object // To understand how etags are calculated, read more here: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html // https://teppen.io/2018/06/23/aws_s3_etags/ - let mut etag_md5_hasher = Md5::new(); + let mut checksummer = MultipartChecksummer::init(checksum_algorithm); for part in parts.iter() { - etag_md5_hasher.update(part.etag.as_ref().unwrap().as_bytes()); + checksummer.update(part.etag.as_ref().unwrap(), part.checksum)?; } - let etag = format!( - "{}-{}", - hex::encode(etag_md5_hasher.finalize()), - parts.len() - ); + let (checksum_md5, checksum_extra) = checksummer.finalize(); + + if expected_checksum.is_some() && checksum_extra != expected_checksum { + return Err(Error::InvalidDigest( + "Failed to validate x-amz-checksum-*".into(), + )); + } + + let etag = format!("{}-{}", hex::encode(&checksum_md5[..]), parts.len()); // Calculate total size of final object let total_size = parts.iter().map(|x| x.size.unwrap()).sum(); - if let Err(e) = check_quotas(&garage, bucket, &key, total_size).await { + if let Err(e) = check_quotas(&ctx, total_size, Some(&object)).await { object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); garage.object_table.insert(&final_object).await?; return Err(e); } + // If there is a checksum algorithm, update metadata with checksum + let object_encryption = match checksum_algorithm { + None => object_encryption, + Some(_) => { + let (encryption, meta) = + EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?; + let new_meta = ObjectVersionMetaInner { + headers: meta.into_owned().headers, + checksum: checksum_extra, + }; + encryption.encrypt_meta(new_meta)? + } + }; + // Write final object version object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( ObjectVersionMeta { - headers, + encryption: object_encryption, size: total_size, etag: etag.clone(), }, final_version.blocks.items()[0].1.hash, )); - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); garage.object_table.insert(&final_object).await?; // Send response saying ok we're done let result = s3_xml::CompleteMultipartUploadResult { xmlns: (), - location: None, + // FIXME: the location returned is not always correct: + // - we always return https, but maybe some people do http + // - if root_domain is not specified, a full URL is not returned + location: garage + .config + .s3_api + .root_domain + .as_ref() + .map(|rd| s3_xml::Value(format!("https://{}.{}/{}", bucket_name, rd, key))) + .or(Some(s3_xml::Value(format!("/{}/{}", bucket_name, key)))), bucket: s3_xml::Value(bucket_name.to_string()), key: s3_xml::Value(key), etag: s3_xml::Value(format!("\"{}\"", etag)), + checksum_crc32: match &checksum_extra { + Some(ChecksumValue::Crc32(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_crc32c: match &checksum_extra { + Some(ChecksumValue::Crc32c(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_sha1: match &checksum_extra { + Some(ChecksumValue::Sha1(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_sha256: match &checksum_extra { + Some(ChecksumValue::Sha256(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, }; let xml = s3_xml::to_xml_with_header(&result)?; - Ok(Response::new(Body::from(xml.into_bytes()))) + let resp = Response::builder(); + let resp = add_checksum_response_headers(&expected_checksum, resp); + Ok(resp.body(string_body(xml))?) } pub async fn handle_abort_multipart_upload( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, key: &str, upload_id: &str, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let upload_id = decode_upload_id(upload_id)?; - let (_, mut object_version, _) = - get_upload(&garage, &bucket_id, &key.to_string(), &upload_id).await?; + let (_, mut object_version, _) = get_upload(&ctx, &key.to_string(), &upload_id).await?; object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket_id, key.to_string(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]); garage.object_table.insert(&final_object).await?; - Ok(Response::new(Body::from(vec![]))) + Ok(Response::new(empty_body())) } // ======== helpers ============ #[allow(clippy::ptr_arg)] pub(crate) async fn get_upload( - garage: &Garage, - bucket_id: &Uuid, + ctx: &ReqCtx, key: &String, upload_id: &Uuid, ) -> Result<(Object, ObjectVersion, MultipartUpload), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let (object, mpu) = futures::try_join!( garage.object_table.get(bucket_id, key).map_err(Error::from), garage @@ -427,6 +546,7 @@ pub fn decode_upload_id(id: &str) -> Result { struct CompleteMultipartUploadPart { etag: String, part_number: u64, + checksum: Option, } fn parse_complete_multipart_upload_body( @@ -452,9 +572,41 @@ fn parse_complete_multipart_upload_body( .children() .find(|e| e.has_tag_name("PartNumber"))? .text()?; + let checksum = if let Some(crc32) = + item.children().find(|e| e.has_tag_name("ChecksumCRC32")) + { + Some(ChecksumValue::Crc32( + BASE64_STANDARD.decode(crc32.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(crc32c) = item.children().find(|e| e.has_tag_name("ChecksumCRC32C")) + { + Some(ChecksumValue::Crc32c( + BASE64_STANDARD.decode(crc32c.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) { + Some(ChecksumValue::Sha1( + BASE64_STANDARD.decode(sha1.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(sha256) = item.children().find(|e| e.has_tag_name("ChecksumSHA256")) + { + Some(ChecksumValue::Sha256( + BASE64_STANDARD.decode(sha256.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else { + None + }; parts.push(CompleteMultipartUploadPart { etag: etag.trim_matches('"').to_string(), part_number: part_number.parse().ok()?, + checksum, }); } else { return None; @@ -463,3 +615,99 @@ fn parse_complete_multipart_upload_body( Some(parts) } + +// ====== checksummer ==== + +#[derive(Default)] +pub(crate) struct MultipartChecksummer { + pub md5: Md5, + pub extra: Option, +} + +pub(crate) enum MultipartExtraChecksummer { + Crc32(Crc32), + Crc32c(Crc32c), + Sha1(Sha1), + Sha256(Sha256), +} + +impl MultipartChecksummer { + pub(crate) fn init(algo: Option) -> Self { + Self { + md5: Md5::new(), + extra: match algo { + None => None, + Some(ChecksumAlgorithm::Crc32) => { + Some(MultipartExtraChecksummer::Crc32(Crc32::new())) + } + Some(ChecksumAlgorithm::Crc32c) => { + Some(MultipartExtraChecksummer::Crc32c(Crc32c::default())) + } + Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())), + Some(ChecksumAlgorithm::Sha256) => { + Some(MultipartExtraChecksummer::Sha256(Sha256::new())) + } + }, + } + } + + pub(crate) fn update( + &mut self, + etag: &str, + checksum: Option, + ) -> Result<(), Error> { + self.md5 + .update(&hex::decode(&etag).ok_or_message("invalid etag hex")?); + match (&mut self.extra, checksum) { + (None, _) => (), + ( + Some(MultipartExtraChecksummer::Crc32(ref mut crc32)), + Some(ChecksumValue::Crc32(x)), + ) => { + crc32.update(&x); + } + ( + Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)), + Some(ChecksumValue::Crc32c(x)), + ) => { + crc32c.write(&x); + } + (Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => { + sha1.update(&x); + } + ( + Some(MultipartExtraChecksummer::Sha256(ref mut sha256)), + Some(ChecksumValue::Sha256(x)), + ) => { + sha256.update(&x); + } + (Some(_), b) => { + return Err(Error::internal_error(format!( + "part checksum was not computed correctly, got: {:?}", + b + ))) + } + } + Ok(()) + } + + pub(crate) fn finalize(self) -> (Md5Checksum, Option) { + let md5 = self.md5.finalize()[..].try_into().unwrap(); + let extra = match self.extra { + None => None, + Some(MultipartExtraChecksummer::Crc32(crc32)) => { + Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize()))) + } + Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c( + u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()), + )), + Some(MultipartExtraChecksummer::Sha1(sha1)) => { + Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap())) + } + Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256( + sha256.finalize()[..].try_into().unwrap(), + )), + }; + (md5, extra) + } +} diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 542b7a81..09be7e7c 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::convert::TryInto; +use std::convert::{Infallible, TryInto}; use std::ops::RangeInclusive; use std::sync::Arc; use std::task::{Context, Poll}; @@ -9,22 +9,29 @@ use bytes::Bytes; use chrono::{DateTime, Duration, Utc}; use futures::{Stream, StreamExt}; use hyper::header::{self, HeaderMap, HeaderName, HeaderValue}; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use multer::{Constraints, Multipart, SizeLimit}; use serde::Deserialize; use garage_model::garage::Garage; +use garage_model::s3::object_table::*; -use crate::s3::error::*; -use crate::s3::put::{get_headers, save_stream}; -use crate::s3::xml as s3_xml; -use crate::signature::payload::{parse_date, verify_v4}; +use garage_api_common::cors::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; + +use crate::api_server::ResBody; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::{extract_metadata_headers, save_stream, ChecksumMode}; +use crate::xml as s3_xml; pub async fn handle_post_object( garage: Arc, - req: Request, + req: Request, bucket_name: String, -) -> Result, Error> { +) -> Result, Error> { let boundary = req .headers() .get(header::CONTENT_TYPE) @@ -41,16 +48,21 @@ pub async fn handle_post_object( ); let (head, body) = req.into_parts(); - let mut multipart = Multipart::with_constraints(body, boundary, constraints); + let stream = body_stream::<_, Error>(body); + let mut multipart = Multipart::with_constraints(stream, boundary, constraints); let mut params = HeaderMap::new(); - let field = loop { + let file_field = loop { let field = if let Some(field) = multipart.next_field().await? { field } else { return Err(Error::bad_request("Request did not contain a file")); }; - let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) { + let name: HeaderName = if let Some(Ok(name)) = field + .name() + .map(str::to_ascii_lowercase) + .map(TryInto::try_into) + { name } else { continue; @@ -60,21 +72,11 @@ pub async fn handle_post_object( } if let Ok(content) = HeaderValue::from_str(&field.text().await?) { - match name.as_str() { - "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), - "acl" => { - if params.insert("x-amz-acl", content).is_some() { - return Err(Error::bad_request("Field 'acl' provided more than once")); - } - } - _ => { - if params.insert(&name, content).is_some() { - return Err(Error::bad_request(format!( - "Field '{}' provided more than once", - name - ))); - } - } + if params.insert(&name, content).is_some() { + return Err(Error::bad_request(format!( + "Field '{}' provided more than once", + name + ))); } } }; @@ -84,26 +86,15 @@ pub async fn handle_post_object( .get("key") .ok_or_bad_request("No key was provided")? .to_str()?; - let credential = params - .get("x-amz-credential") - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? - .to_str()?; let policy = params .get("policy") .ok_or_bad_request("No policy was provided")? .to_str()?; - let signature = params - .get("x-amz-signature") - .ok_or_bad_request("No signature was provided")? - .to_str()?; - let date = params - .get("x-amz-date") - .ok_or_bad_request("No date was provided")? - .to_str()?; + let authorization = Authorization::parse_form(¶ms)?; let key = if key.contains("${filename}") { // if no filename is provided, don't replace. This matches the behavior of AWS. - if let Some(filename) = field.file_name() { + if let Some(filename) = file_field.file_name() { key.replace("${filename}", filename) } else { key.to_owned() @@ -112,21 +103,13 @@ pub async fn handle_post_object( key.to_owned() }; - let date = parse_date(date)?; - let api_key = verify_v4( - &garage, - "s3", - credential, - &date, - signature, - policy.as_bytes(), - ) - .await?; + let api_key = verify_v4(&garage, "s3", &authorization, policy.as_bytes()).await?; let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; if !api_key.allow_write(&bucket_id) { return Err(Error::forbidden("Operation is not allowed for this key.")); @@ -136,6 +119,12 @@ pub async fn handle_post_object( .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); + let matching_cors_rule = find_matching_cors_rule( + &bucket_params, + &Request::from_parts(head.clone(), empty_body::()), + )? + .cloned(); let decoded_policy = BASE64_STANDARD .decode(policy) @@ -152,11 +141,26 @@ pub async fn handle_post_object( let mut conditions = decoded_policy.into_conditions()?; + // If there are conditions on the bucket name, check these against the actual bucket_name rather + // than the one in params, which is allowed to be absent. + if let Some(conds) = conditions.params.remove("bucket") { + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s.as_str() == bucket_name, + Operation::StartsWith(s) => bucket_name.starts_with(&s), + }; + if !ok { + return Err(Error::bad_request( + "Key 'bucket' has value not allowed in policy", + )); + } + } + } + for (param_key, value) in params.iter() { - let mut param_key = param_key.to_string(); - param_key.make_ascii_lowercase(); - match param_key.as_str() { - "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields + let param_key = param_key.as_str(); + match param_key { + "policy" | "x-amz-signature" | "bucket" => (), // this is always accepted, as it's required to validate other fields "content-type" => { let conds = conditions.params.remove("content-type").ok_or_else(|| { Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) @@ -200,7 +204,7 @@ pub async fn handle_post_object( // how aws seems to behave. continue; } - let conds = conditions.params.remove(¶m_key).ok_or_else(|| { + let conds = conditions.params.remove(param_key).ok_or_else(|| { Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { @@ -226,23 +230,52 @@ pub async fn handle_post_object( ))); } - let headers = get_headers(¶ms)?; + // if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere + // around here to make sure the rest of the machinery takes our acl into account. + let headers = extract_metadata_headers(¶ms)?; - let stream = field.map(|r| r.map_err(Into::into)); - let (_, md5) = save_stream( - garage, + let checksum_algorithm = request_checksum_algorithm(¶ms)?; + let expected_checksums = ExpectedChecksums { + md5: params + .get("content-md5") + .map(HeaderValue::to_str) + .transpose()? + .map(str::to_string), + sha256: None, + extra: checksum_algorithm + .map(|algo| extract_checksum_value(¶ms, algo)) + .transpose()?, + }; + + let meta = ObjectVersionMetaInner { headers, + checksum: expected_checksums.extra, + }; + + let encryption = EncryptionParams::new_from_headers(&garage, ¶ms)?; + + let stream = file_field.map(|r| r.map_err(Into::into)); + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + + let res = save_stream( + &ctx, + meta, + encryption, StreamLimiter::new(stream, conditions.content_length), - &bucket, &key, - None, - None, + ChecksumMode::Verify(&expected_checksums), ) .await?; - let etag = format!("\"{}\"", md5); + let etag = format!("\"{}\"", res.etag); - let resp = if let Some(mut target) = params + let mut resp = if let Some(mut target) = params .get("success_action_redirect") .and_then(|h| h.to_str().ok()) .and_then(|u| url::Url::parse(u).ok()) @@ -250,20 +283,20 @@ pub async fn handle_post_object( { target .query_pairs_mut() - .append_pair("bucket", &bucket_name) + .append_pair("bucket", &ctx.bucket_name) .append_pair("key", &key) .append_pair("etag", &etag); let target = target.to_string(); - Response::builder() + let mut resp = Response::builder() .status(StatusCode::SEE_OTHER) .header(header::LOCATION, target.clone()) - .header(header::ETAG, etag) - .body(target.into())? + .header(header::ETAG, etag); + encryption.add_response_headers(&mut resp); + resp.body(string_body(target))? } else { let path = head .uri - .into_parts() - .path_and_query + .path_and_query() .map(|paq| paq.path().to_string()) .unwrap_or_else(|| "/".to_string()); let authority = head @@ -286,28 +319,34 @@ pub async fn handle_post_object( .get("success_action_status") .and_then(|h| h.to_str().ok()) .unwrap_or("204"); - let builder = Response::builder() + let mut builder = Response::builder() .header(header::LOCATION, location.clone()) .header(header::ETAG, etag.clone()); + encryption.add_response_headers(&mut builder); match action { - "200" => builder.status(StatusCode::OK).body(Body::empty())?, + "200" => builder.status(StatusCode::OK).body(empty_body())?, "201" => { let xml = s3_xml::PostObject { xmlns: (), location: s3_xml::Value(location), - bucket: s3_xml::Value(bucket_name), + bucket: s3_xml::Value(ctx.bucket_name), key: s3_xml::Value(key), etag: s3_xml::Value(etag), }; let body = s3_xml::to_xml_with_header(&xml)?; builder .status(StatusCode::CREATED) - .body(Body::from(body.into_bytes()))? + .body(string_body(body))? } - _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, + _ => builder.status(StatusCode::NO_CONTENT).body(empty_body())?, } }; + if let Some(rule) = matching_cors_rule { + add_cors_headers(&mut resp, &rule) + .ok_or_internal_error("Invalid bucket CORS configuration")?; + } + Ok(resp) } diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index c7ac5030..b915f2ec 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -1,121 +1,212 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::sync::Arc; -use base64::prelude::*; use futures::prelude::*; -use hyper::body::{Body, Bytes}; +use futures::stream::FuturesOrdered; +use futures::try_join; + +use tokio::sync::mpsc; + +use hyper::body::Bytes; use hyper::header::{HeaderMap, HeaderValue}; use hyper::{Request, Response}; -use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; -use sha2::Sha256; use opentelemetry::{ trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, Context, }; -use garage_rpc::netapp::bytes_buf::BytesBuf; +use garage_net::bytes_buf::BytesBuf; +use garage_rpc::rpc_helper::OrderTag; use garage_table::*; -use garage_util::async_hash::*; use garage_util::data::*; use garage_util::error::Error as GarageError; use garage_util::time::*; use garage_block::manager::INLINE_THRESHOLD; -use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::index_counter::CountedItem; use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::s3::error::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::body::StreamingChecksumReceiver; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; + +pub(crate) struct SaveStreamResult { + pub(crate) version_uuid: Uuid, + pub(crate) version_timestamp: u64, + /// Etag WITHOUT THE QUOTES (just the hex value) + pub(crate) etag: String, +} + +pub(crate) enum ChecksumMode<'a> { + Verify(&'a ExpectedChecksums), + VerifyFrom { + checksummer: StreamingChecksumReceiver, + trailer_algo: Option, + }, + Calculate(Option), +} pub async fn handle_put( - garage: Arc, - req: Request, - bucket: &Bucket, - key: &str, - content_sha256: Option, -) -> Result, Error> { + ctx: ReqCtx, + req: Request, + key: &String, +) -> Result, Error> { // Retrieve interesting headers from request - let headers = get_headers(req.headers())?; + let headers = extract_metadata_headers(req.headers())?; debug!("Object headers: {:?}", headers); - let content_md5 = match req.headers().get("content-md5") { - Some(x) => Some(x.to_str()?.to_string()), - None => None, + let expected_checksums = ExpectedChecksums { + md5: match req.headers().get("content-md5") { + Some(x) => Some(x.to_str()?.to_string()), + None => None, + }, + sha256: None, + extra: request_checksum_value(req.headers())?, + }; + let trailer_checksum_algorithm = request_trailer_checksum_algorithm(req.headers())?; + + let meta = ObjectVersionMetaInner { + headers, + checksum: expected_checksums.extra, }; - let (_head, body) = req.into_parts(); - let body = body.map_err(Error::from); + // Determine whether object should be encrypted, and if so the key + let encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?; - save_stream( - garage, - headers, - body, - bucket, + // The request body is a special ReqBody object (see garage_api_common::signature::body) + // which supports calculating checksums while streaming the data. + // Before we start streaming, we configure it to calculate all the checksums we need. + let mut req_body = req.into_body(); + req_body.add_expected_checksums(expected_checksums.clone()); + if !encryption.is_encrypted() { + // For non-encrypted objects, we need to compute the md5sum in all cases + // (even if content-md5 is not set), because it is used as the object etag + req_body.add_md5(); + } + + let (stream, checksummer) = req_body.streaming_with_checksums(); + let stream = stream.map_err(Error::from); + + let res = save_stream( + &ctx, + meta, + encryption, + stream, key, - content_md5, - content_sha256, + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo: trailer_checksum_algorithm, + }, ) - .await - .map(|(uuid, md5)| put_response(uuid, md5)) + .await?; + + let mut resp = Response::builder() + .header("x-amz-version-id", hex::encode(res.version_uuid)) + .header("ETag", format!("\"{}\"", res.etag)); + encryption.add_response_headers(&mut resp); + let resp = add_checksum_response_headers(&expected_checksums.extra, resp); + Ok(resp.body(empty_body())?) } pub(crate) async fn save_stream> + Unpin>( - garage: Arc, - headers: ObjectVersionHeaders, + ctx: &ReqCtx, + mut meta: ObjectVersionMetaInner, + encryption: EncryptionParams, body: S, - bucket: &Bucket, - key: &str, - content_md5: Option, - content_sha256: Option, -) -> Result<(Uuid, String), Error> { - // Generate identity of new version - let version_uuid = gen_uuid(); - let version_timestamp = now_msec(); + key: &String, + checksum_mode: ChecksumMode<'_>, +) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let mut chunker = StreamChunker::new(body, garage.config.block_size); - let first_block = chunker.next().await?.unwrap_or_default(); + let (first_block_opt, existing_object) = try_join!( + chunker.next(), + garage.object_table.get(bucket_id, key).map_err(Error::from), + )?; + + let first_block = first_block_opt.unwrap_or_default(); + + // Generate identity of new version + let version_uuid = gen_uuid(); + let version_timestamp = next_timestamp(existing_object.as_ref()); + + let mut checksummer = match &checksum_mode { + ChecksumMode::Verify(expected) => Checksummer::init(expected, !encryption.is_encrypted()), + ChecksumMode::Calculate(algo) => { + Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(*algo) + } + ChecksumMode::VerifyFrom { .. } => { + // Checksums are calculated by the garage_api_common::signature module + // so here we can just have an empty checksummer that does nothing + Checksummer::new() + } + }; // If body is small enough, store it directly in the object table // as "inline data". We can then return immediately. if first_block.len() < INLINE_THRESHOLD { - let mut md5sum = Md5::new(); - md5sum.update(&first_block[..]); - let data_md5sum = md5sum.finalize(); - let data_md5sum_hex = hex::encode(data_md5sum); + checksummer.update(&first_block); + let mut checksums = checksummer.finalize(); + + match checksum_mode { + ChecksumMode::Verify(expected) => { + checksums.verify(&expected)?; + } + ChecksumMode::Calculate(algo) => { + meta.checksum = checksums.extract(algo); + } + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo, + } => { + drop(chunker); + checksums = checksummer + .await + .ok_or_internal_error("checksum calculation")??; + if let Some(algo) = trailer_algo { + meta.checksum = checksums.extract(Some(algo)); + } + } + }; - let data_sha256sum = sha256sum(&first_block[..]); let size = first_block.len() as u64; + check_quotas(ctx, size, existing_object.as_ref()).await?; - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; - - check_quotas(&garage, bucket, key, size).await?; + let etag = encryption.etag_from_md5(&checksums.md5); + let inline_data = encryption.encrypt_blob(&first_block)?.to_vec(); let object_version = ObjectVersion { uuid: version_uuid, timestamp: version_timestamp, state: ObjectVersionState::Complete(ObjectVersionData::Inline( ObjectVersionMeta { - headers, + encryption: encryption.encrypt_meta(meta)?, size, - etag: data_md5sum_hex.clone(), + etag: etag.clone(), }, - first_block.to_vec(), + inline_data, )), }; - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - return Ok((version_uuid, data_md5sum_hex)); + return Ok(SaveStreamResult { + version_uuid, + version_timestamp, + etag, + }); } // The following consists in many steps that can each fail. @@ -123,7 +214,7 @@ pub(crate) async fn save_stream> + Unpin>( // before everything is finished (cleanup is done using the Drop trait). let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner { garage: garage.clone(), - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.into(), version_uuid, version_timestamp, @@ -135,11 +226,12 @@ pub(crate) async fn save_stream> + Unpin>( uuid: version_uuid, timestamp: version_timestamp, state: ObjectVersionState::Uploading { - headers: headers.clone(), + encryption: encryption.encrypt_meta(meta.clone())?, + checksum_algorithm: None, // don't care; overwritten later multipart: false, }, }; - let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]); + let object = Object::new(*bucket_id, key.into(), vec![object_version.clone()]); garage.object_table.insert(&object).await?; // Initialize corresponding entry in version table @@ -149,102 +241,100 @@ pub(crate) async fn save_stream> + Unpin>( let version = Version::new( version_uuid, VersionBacklink::Object { - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.into(), }, false, ); garage.version_table.insert(&version).await?; - // Transfer data and verify checksum - let first_block_hash = async_blake2sum(first_block.clone()).await; - - let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks( - &garage, + // Transfer data + let (total_size, mut checksums, first_block_hash) = read_and_put_blocks( + ctx, &version, + encryption, 1, first_block, - first_block_hash, - &mut chunker, + chunker, + checksummer, ) .await?; - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; + // Verify checksums are ok / add calculated checksum to metadata + match checksum_mode { + ChecksumMode::Verify(expected) => { + checksums.verify(&expected)?; + } + ChecksumMode::Calculate(algo) => { + meta.checksum = checksums.extract(algo); + } + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo, + } => { + checksums = checksummer + .await + .ok_or_internal_error("checksum calculation")??; + if let Some(algo) = trailer_algo { + meta.checksum = checksums.extract(Some(algo)); + } + } + }; - check_quotas(&garage, bucket, key, total_size).await?; + // Verify quotas are respsected + check_quotas(ctx, total_size, existing_object.as_ref()).await?; // Save final object state, marked as Complete - let md5sum_hex = hex::encode(data_md5sum); + let etag = encryption.etag_from_md5(&checksums.md5); + object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( ObjectVersionMeta { - headers, + encryption: encryption.encrypt_meta(meta)?, size: total_size, - etag: md5sum_hex.clone(), + etag: etag.clone(), }, first_block_hash, )); - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; // We were not interrupted, everything went fine. // We won't have to clean up on drop. interrupted_cleanup.cancel(); - Ok((version_uuid, md5sum_hex)) -} - -/// Validate MD5 sum against content-md5 header -/// and sha256sum against signed content-sha256 -pub(crate) fn ensure_checksum_matches( - data_md5sum: &[u8], - data_sha256sum: garage_util::data::FixedBytes32, - content_md5: Option<&str>, - content_sha256: Option, -) -> Result<(), Error> { - if let Some(expected_sha256) = content_sha256 { - if expected_sha256 != data_sha256sum { - return Err(Error::bad_request( - "Unable to validate x-amz-content-sha256", - )); - } else { - trace!("Successfully validated x-amz-content-sha256"); - } - } - if let Some(expected_md5) = content_md5 { - if expected_md5.trim_matches('"') != BASE64_STANDARD.encode(data_md5sum) { - return Err(Error::bad_request("Unable to validate content-md5")); - } else { - trace!("Successfully validated content-md5"); - } - } - Ok(()) + Ok(SaveStreamResult { + version_uuid, + version_timestamp, + etag, + }) } /// Check that inserting this object with this size doesn't exceed bucket quotas pub(crate) async fn check_quotas( - garage: &Arc, - bucket: &Bucket, - key: &str, + ctx: &ReqCtx, size: u64, + prev_object: Option<&Object>, ) -> Result<(), Error> { - let quotas = bucket.state.as_option().unwrap().quotas.get(); + let ReqCtx { + garage, + bucket_id, + bucket_params, + .. + } = ctx; + + let quotas = bucket_params.quotas.get(); if quotas.max_objects.is_none() && quotas.max_size.is_none() { return Ok(()); }; - let key = key.to_string(); - let (prev_object, counters) = futures::try_join!( - garage.object_table.get(&bucket.id, &key), - garage.object_counter_table.table.get(&bucket.id, &EmptyKey), - )?; + let counters = garage + .object_counter_table + .table + .get(bucket_id, &EmptyKey) + .await?; let counters = counters - .map(|x| x.filtered_values(&garage.system.ring.borrow())) + .map(|x| x.filtered_values(&garage.system.cluster_layout())) .unwrap_or_default(); let (prev_cnt_obj, prev_cnt_size) = match prev_object { @@ -275,7 +365,7 @@ pub(crate) async fn check_quotas( if cnt_size_diff > 0 && current_size + cnt_size_diff > ms as i64 { return Err(Error::forbidden(format!( "Bucket size quota is reached, maximum total size of objects for this bucket: {}. The bucket is already {} bytes, and this object would add {} bytes.", - ms, current_size, size + ms, current_size, cnt_size_diff ))); } } @@ -284,89 +374,185 @@ pub(crate) async fn check_quotas( } pub(crate) async fn read_and_put_blocks> + Unpin>( - garage: &Garage, + ctx: &ReqCtx, version: &Version, + encryption: EncryptionParams, part_number: u64, first_block: Bytes, - first_block_hash: Hash, - chunker: &mut StreamChunker, -) -> Result<(u64, GenericArray, Hash), Error> { + mut chunker: StreamChunker, + checksummer: Checksummer, +) -> Result<(u64, Checksums, Hash), Error> { let tracer = opentelemetry::global::tracer("garage"); - let md5hasher = AsyncHasher::::new(); - let sha256hasher = AsyncHasher::::new(); + let (block_tx, mut block_rx) = mpsc::channel::>(2); + let read_blocks = async { + block_tx.send(Ok(first_block)).await?; + loop { + let res = chunker + .next() + .with_context(Context::current_with_span( + tracer.start("Read block from client"), + )) + .await; + match res { + Ok(Some(block)) => block_tx.send(Ok(block)).await?, + Ok(None) => break, + Err(e) => { + block_tx.send(Err(e)).await?; + break; + } + } + } + drop(block_tx); + Ok::<_, mpsc::error::SendError<_>>(()) + }; - futures::future::join( - md5hasher.update(first_block.clone()), - sha256hasher.update(first_block.clone()), - ) - .with_context(Context::current_with_span( - tracer.start("Hash first block (md5, sha256)"), - )) - .await; + let (block_tx2, mut block_rx2) = mpsc::channel::>(1); + let hash_stream = async { + let mut checksummer = checksummer; + while let Some(next) = block_rx.recv().await { + match next { + Ok(block) => { + block_tx2.send(Ok(block.clone())).await?; + checksummer = tokio::task::spawn_blocking(move || { + checksummer.update(&block); + checksummer + }) + .with_context(Context::current_with_span( + tracer.start("Hash block (md5, sha256)"), + )) + .await + .unwrap() + } + Err(e) => { + block_tx2.send(Err(e)).await?; + break; + } + } + } + drop(block_tx2); + Ok::<_, mpsc::error::SendError<_>>(checksummer) + }; - let mut next_offset = first_block.len(); - let mut put_curr_version_block = put_block_meta( - garage, - version, - part_number, - 0, - first_block_hash, - first_block.len() as u64, - ); - let mut put_curr_block = garage - .block_manager - .rpc_put_block(first_block_hash, first_block); + let (block_tx3, mut block_rx3) = mpsc::channel::>(1); + let encrypt_hash_blocks = async { + let mut first_block_hash = None; + while let Some(next) = block_rx2.recv().await { + match next { + Ok(block) => { + let unencrypted_len = block.len() as u64; + let res = tokio::task::spawn_blocking(move || { + let block = encryption.encrypt_block(block)?; + let hash = blake2sum(&block); + Ok((block, hash)) + }) + .with_context(Context::current_with_span( + tracer.start("Encrypt and hash (blake2) block"), + )) + .await + .unwrap(); + match res { + Ok((block, hash)) => { + if first_block_hash.is_none() { + first_block_hash = Some(hash); + } + block_tx3.send(Ok((block, unencrypted_len, hash))).await?; + } + Err(e) => { + block_tx3.send(Err(e)).await?; + break; + } + } + } + Err(e) => { + block_tx3.send(Err(e)).await?; + break; + } + } + } + drop(block_tx3); + Ok::<_, mpsc::error::SendError<_>>(first_block_hash.unwrap()) + }; - loop { - let (_, _, next_block) = futures::try_join!( - put_curr_block.map_err(Error::from), - put_curr_version_block.map_err(Error::from), - chunker.next(), - )?; - if let Some(block) = next_block { - let (_, _, block_hash) = futures::future::join3( - md5hasher.update(block.clone()), - sha256hasher.update(block.clone()), - async_blake2sum(block.clone()), - ) - .with_context(Context::current_with_span( - tracer.start("Hash block (md5, sha256, blake2)"), - )) - .await; - let block_len = block.len(); - put_curr_version_block = put_block_meta( - garage, + let put_blocks = async { + // Structure for handling several concurrent writes to storage nodes + let order_stream = OrderTag::stream(); + let mut write_futs = FuturesOrdered::new(); + let mut written_bytes = 0u64; + loop { + // Simultaneously write blocks to storage nodes & await for next block to be written + let currently_running = write_futs.len(); + let write_futs_next = async { + if write_futs.is_empty() { + futures::future::pending().await + } else { + write_futs.next().await.unwrap() + } + }; + let recv_next = async { + // If more than a maximum number of writes are in progress, don't add more for now + if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request { + futures::future::pending().await + } else { + block_rx3.recv().await + } + }; + let (block, unencrypted_len, hash) = tokio::select! { + result = write_futs_next => { + result?; + continue; + }, + recv = recv_next => match recv { + Some(next) => next?, + None => break, + }, + }; + + // For next block to be written: count its size and spawn future to write it + write_futs.push_back(put_block_and_meta( + ctx, version, part_number, - next_offset as u64, - block_hash, - block_len as u64, - ); - put_curr_block = garage.block_manager.rpc_put_block(block_hash, block); - next_offset += block_len; - } else { - break; + written_bytes, + hash, + block, + unencrypted_len, + encryption.is_encrypted(), + order_stream.order(written_bytes), + )); + written_bytes += unencrypted_len; } - } + while let Some(res) = write_futs.next().await { + res?; + } + Ok::<_, Error>(written_bytes) + }; - let total_size = next_offset as u64; - let data_md5sum = md5hasher.finalize().await; + let (_, stream_hash_result, block_hash_result, final_result) = + futures::join!(read_blocks, hash_stream, encrypt_hash_blocks, put_blocks); - let data_sha256sum = sha256hasher.finalize().await; - let data_sha256sum = Hash::try_from(&data_sha256sum[..]).unwrap(); + let total_size = final_result?; + // unwrap here is ok, because if hasher failed, it is because something failed + // later in the pipeline which already caused a return at the ? on previous line + let first_block_hash = block_hash_result.unwrap(); + let checksums = stream_hash_result.unwrap().finalize(); - Ok((total_size, data_md5sum, data_sha256sum)) + Ok((total_size, checksums, first_block_hash)) } -async fn put_block_meta( - garage: &Garage, +async fn put_block_and_meta( + ctx: &ReqCtx, version: &Version, part_number: u64, offset: u64, hash: Hash, + block: Bytes, size: u64, + is_encrypted: bool, + order_tag: OrderTag, ) -> Result<(), GarageError> { + let ReqCtx { garage, .. } = ctx; + let mut version = version.clone(); version.blocks.put( VersionBlockKey { @@ -383,6 +569,9 @@ async fn put_block_meta( }; futures::try_join!( + garage + .block_manager + .rpc_put_block(hash, block, is_encrypted, Some(order_tag)), garage.version_table.insert(&version), garage.block_ref_table.insert(&block_ref), )?; @@ -425,14 +614,6 @@ impl> + Unpin> StreamChunker { } } -pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response { - Response::builder() - .header("x-amz-version-id", hex::encode(version_uuid)) - .header("ETag", format!("\"{}\"", md5sum_hex)) - .body(Body::from(vec![])) - .unwrap() -} - struct InterruptedCleanup(Option); struct InterruptedCleanupInner { garage: Arc, @@ -467,55 +648,55 @@ impl Drop for InterruptedCleanup { // ============ helpers ============ -pub(crate) fn get_mime_type(headers: &HeaderMap) -> Result { - Ok(headers - .get(hyper::header::CONTENT_TYPE) - .map(|x| x.to_str()) - .unwrap_or(Ok("blob"))? - .to_string()) -} - -pub(crate) fn get_headers(headers: &HeaderMap) -> Result { - let content_type = get_mime_type(headers)?; - let mut other = BTreeMap::new(); +pub(crate) fn extract_metadata_headers( + headers: &HeaderMap, +) -> Result { + let mut ret = Vec::new(); // Preserve standard headers let standard_header = vec![ + hyper::header::CONTENT_TYPE, hyper::header::CACHE_CONTROL, hyper::header::CONTENT_DISPOSITION, hyper::header::CONTENT_ENCODING, hyper::header::CONTENT_LANGUAGE, hyper::header::EXPIRES, ]; - for h in standard_header.iter() { - if let Some(v) = headers.get(h) { - match v.to_str() { - Ok(v_str) => { - other.insert(h.to_string(), v_str.to_string()); - } - Err(e) => { - warn!("Discarding header {}, error in .to_str(): {}", h, e); - } - } + for name in standard_header.iter() { + if let Some(value) = headers.get(name) { + ret.push((name.to_string(), value.to_str()?.to_string())); } } // Preserve x-amz-meta- headers - for (k, v) in headers.iter() { - if k.as_str().starts_with("x-amz-meta-") { - match v.to_str() { - Ok(v_str) => { - other.insert(k.to_string(), v_str.to_string()); - } - Err(e) => { - warn!("Discarding header {}, error in .to_str(): {}", k, e); - } + for (name, value) in headers.iter() { + if name.as_str().starts_with("x-amz-meta-") { + ret.push(( + name.as_str().to_ascii_lowercase(), + std::str::from_utf8(value.as_bytes())?.to_string(), + )); + } + if name == X_AMZ_WEBSITE_REDIRECT_LOCATION { + let value = std::str::from_utf8(value.as_bytes())?.to_string(); + if !(value.starts_with("/") + || value.starts_with("http://") + || value.starts_with("https://")) + { + return Err(Error::bad_request(format!( + "Invalid {X_AMZ_WEBSITE_REDIRECT_LOCATION} header", + ))); } + ret.push((X_AMZ_WEBSITE_REDIRECT_LOCATION.to_string(), value)); } } - Ok(ObjectVersionHeaders { - content_type, - other, - }) + Ok(ret) +} + +pub(crate) fn next_timestamp(existing_object: Option<&Object>) -> u64 { + existing_object + .as_ref() + .and_then(|obj| obj.versions().iter().map(|v| v.timestamp).max()) + .map(|t| std::cmp::max(t + 1, now_msec())) + .unwrap_or_else(now_msec) } diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 821b0e07..e3f58490 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,9 +3,10 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; -use crate::s3::error::*; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; + +use crate::error::*; router_match! {@func @@ -125,6 +126,12 @@ pub enum Endpoint { key: String, part_number: Option, version_id: Option, + response_cache_control: Option, + response_content_disposition: Option, + response_content_encoding: Option, + response_content_language: Option, + response_content_type: Option, + response_expires: Option, }, GetObjectAcl { key: String, @@ -170,7 +177,7 @@ pub enum Endpoint { }, ListBuckets, ListMultipartUploads { - delimiter: Option, + delimiter: Option, encoding_type: Option, key_marker: Option, max_uploads: Option, @@ -178,7 +185,7 @@ pub enum Endpoint { upload_id_marker: Option, }, ListObjects { - delimiter: Option, + delimiter: Option, encoding_type: Option, marker: Option, max_keys: Option, @@ -188,7 +195,7 @@ pub enum Endpoint { // This value should always be 2. It is not checked when constructing the struct list_type: String, continuation_token: Option, - delimiter: Option, + delimiter: Option, encoding_type: Option, fetch_owner: Option, max_keys: Option, @@ -196,7 +203,7 @@ pub enum Endpoint { start_after: Option, }, ListObjectVersions { - delimiter: Option, + delimiter: Option, encoding_type: Option, key_marker: Option, max_keys: Option, @@ -345,6 +352,18 @@ impl Endpoint { _ => return Err(Error::bad_request("Unknown method")), }; + if let Some(x_id) = query.x_id.take() { + if x_id != res.name() { + // I think AWS ignores the x-id parameter. + // Let's make this at least be a warnin to help debugging. + warn!( + "x-id ({}) does not match parsed endpoint ({})", + x_id, + res.name() + ); + } + } + if let Some(message) = query.nonempty_message() { debug!("Unused query parameter: {}", message) } @@ -358,7 +377,14 @@ impl Endpoint { (query.keyword.take().unwrap_or_default(), key, query, None), key: [ EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker), - EMPTY => GetObject (query_opt::version_id, opt_parse::part_number), + EMPTY => GetObject (query_opt::version_id, + opt_parse::part_number, + query_opt::response_cache_control, + query_opt::response_content_disposition, + query_opt::response_content_encoding, + query_opt::response_content_language, + query_opt::response_content_type, + query_opt::response_expires), ACL => GetObjectAcl (query_opt::version_id), LEGAL_HOLD => GetObjectLegalHold (query_opt::version_id), RETENTION => GetObjectRetention (query_opt::version_id), @@ -671,12 +697,19 @@ generateQueryParameters! { "partNumber" => part_number, "part-number-marker" => part_number_marker, "prefix" => prefix, + "response-cache-control" => response_cache_control, + "response-content-disposition" => response_content_disposition, + "response-content-encoding" => response_content_encoding, + "response-content-language" => response_content_language, + "response-content-type" => response_content_type, + "response-expires" => response_expires, "select-type" => select_type, "start-after" => start_after, "uploadId" => upload_id, "upload-id-marker" => upload_id_marker, "versionId" => version_id, - "version-id-marker" => version_id_marker + "version-id-marker" => version_id_marker, + "x-id" => x_id ] } diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 7f2ab925..03cc01d8 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -1,23 +1,22 @@ use quick_xml::de::from_reader; -use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{header::HeaderName, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; - use garage_model::bucket_table::*; -use garage_model::garage::Garage; -use garage_util::data::*; -pub async fn handle_get_website(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +use garage_api_common::helpers::*; - if let Some(website) = param.website_config.get() { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; + +pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName = + HeaderName::from_static("x-amz-website-redirect-location"); + +pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(website) = bucket_params.website_config.get() { let wc = WebsiteConfiguration { xmlns: (), error_document: website.error_document.as_ref().map(|v| Key { @@ -33,57 +32,59 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result, Error Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } else { Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } } -pub async fn handle_delete_website( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.website_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.website_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_put_website( - garage: Arc, - mut bucket: Bucket, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let body = req.into_body().collect().await?; let conf: WebsiteConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .website_config .update(Some(conf.into_garage_website_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) - .body(Body::empty())?) + .body(empty_body())?) } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] @@ -271,7 +272,7 @@ impl Redirect { return Err(Error::bad_request("Bad XML: invalid protocol")); } } - // TODO there are probably more invalide cases, but which ones? + // TODO there are probably more invalid cases, but which ones? Ok(()) } } diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 06f11288..fdb36318 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,7 +1,7 @@ use quick_xml::se::to_string; use serde::{Deserialize, Serialize, Serializer}; -use crate::s3::error::Error as ApiError; +use crate::error::Error as ApiError; pub fn to_xml_with_header(x: &T) -> Result { let mut xml = r#""#.to_string(); @@ -13,6 +13,10 @@ pub fn xmlns_tag(_v: &(), s: S) -> Result { s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") } +pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { + s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] pub struct Value(#[serde(rename = "$value")] pub String); @@ -131,6 +135,14 @@ pub struct CompleteMultipartUploadResult { pub key: Value, #[serde(rename = "ETag")] pub etag: Value, + #[serde(rename = "ChecksumCRC32")] + pub checksum_crc32: Option, + #[serde(rename = "ChecksumCRC32C")] + pub checksum_crc32c: Option, + #[serde(rename = "ChecksumSHA1")] + pub checksum_sha1: Option, + #[serde(rename = "ChecksumSHA256")] + pub checksum_sha256: Option, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -197,6 +209,14 @@ pub struct PartItem { pub part_number: IntValue, #[serde(rename = "Size")] pub size: IntValue, + #[serde(rename = "ChecksumCRC32")] + pub checksum_crc32: Option, + #[serde(rename = "ChecksumCRC32C")] + pub checksum_crc32c: Option, + #[serde(rename = "ChecksumSHA1")] + pub checksum_sha1: Option, + #[serde(rename = "ChecksumSHA256")] + pub checksum_sha256: Option, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -303,6 +323,42 @@ pub struct PostObject { pub etag: Value, } +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grantee { + #[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")] + pub xmlns_xsi: (), + #[serde(rename = "xsi:type")] + pub typ: String, + #[serde(rename = "DisplayName")] + pub display_name: Option, + #[serde(rename = "ID")] + pub id: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grant { + #[serde(rename = "Grantee")] + pub grantee: Grantee, + #[serde(rename = "Permission")] + pub permission: Value, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlList { + #[serde(rename = "Grant")] + pub entries: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlPolicy { + #[serde(serialize_with = "xmlns_tag")] + pub xmlns: (), + #[serde(rename = "Owner")] + pub owner: Option, + #[serde(rename = "AccessControlList")] + pub acl: AccessControlList, +} + #[cfg(test)] mod tests { use super::*; @@ -411,6 +467,43 @@ mod tests { Ok(()) } + #[test] + fn get_bucket_acl_result() -> Result<(), ApiError> { + let grant = Grant { + grantee: Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(Value("owner_name".to_string())), + id: Some(Value("qsdfjklm".to_string())), + }, + permission: Value("FULL_CONTROL".to_string()), + }; + + let get_bucket_acl = AccessControlPolicy { + xmlns: (), + owner: None, + acl: AccessControlList { + entries: vec![grant], + }, + }; + assert_eq!( + to_xml_with_header(&get_bucket_acl)?, + "\ +\ + \ + \ + \ + owner_name\ + qsdfjklm\ + \ + FULL_CONTROL\ + \ + \ +" + ); + Ok(()) + } + #[test] fn delete_result() -> Result<(), ApiError> { let delete_result = DeleteResult { @@ -500,6 +593,10 @@ mod tests { bucket: Value("mybucket".to_string()), key: Value("a/plop".to_string()), etag: Value("\"3858f62230ac3c915f300c664312c11f-9\"".to_string()), + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: Some(Value("ZJAnHyG8PeKz9tI8UTcHrJos39A=".into())), + checksum_sha256: None, }; assert_eq!( to_xml_with_header(&result)?, @@ -509,6 +606,7 @@ mod tests { mybucket\ a/plop\ "3858f62230ac3c915f300c664312c11f-9"\ + ZJAnHyG8PeKz9tI8UTcHrJos39A=\ " ); Ok(()) @@ -780,12 +878,22 @@ mod tests { last_modified: Value("2010-11-10T20:48:34.000Z".to_string()), part_number: IntValue(2), size: IntValue(10485760), + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha256: Some(Value( + "5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=".into(), + )), + checksum_sha1: None, }, PartItem { etag: Value("\"aaaa18db4cc2f85cedef654fccc4a4x8\"".to_string()), last_modified: Value("2010-11-10T20:48:33.000Z".to_string()), part_number: IntValue(3), size: IntValue(10485760), + checksum_sha256: None, + checksum_crc32c: None, + checksum_crc32: Some(Value("ZJAnHyG8=".into())), + checksum_sha1: None, }, ], initiator: Initiator { @@ -820,12 +928,14 @@ mod tests { 2010-11-10T20:48:34.000Z\ 2\ 10485760\ + 5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=\ \ \ "aaaa18db4cc2f85cedef654fccc4a4x8"\ 2010-11-10T20:48:33.000Z\ 3\ 10485760\ + ZJAnHyG8=\ \ \ umat-user-11116a31-17b5-4fb7-9df5-b288870f11xx\ diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs deleted file mode 100644 index 4b8b990f..00000000 --- a/src/api/signature/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -use chrono::{DateTime, Utc}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -use garage_util::data::{sha256sum, Hash}; - -pub mod error; -pub mod payload; -pub mod streaming; - -use error::*; - -pub const SHORT_DATE: &str = "%Y%m%d"; -pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; - -type HmacSha256 = Hmac; - -pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { - if expected_sha256 != sha256sum(body) { - return Err(Error::bad_request( - "Request content hash does not match signed hash".to_string(), - )); - } - Ok(()) -} - -pub fn signing_hmac( - datetime: &DateTime, - secret_key: &str, - region: &str, - service: &str, -) -> Result { - let secret = String::from("AWS4") + secret_key; - let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; - date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); - let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; - region_hmac.update(region.as_bytes()); - let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; - service_hmac.update(service.as_bytes()); - let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; - signing_hmac.update(b"aws4_request"); - let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; - Ok(hmac) -} - -pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { - format!( - "{}/{}/{}/aws4_request", - datetime.format(SHORT_DATE), - region, - service - ) -} diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs deleted file mode 100644 index b50fb3bb..00000000 --- a/src/api/signature/payload.rs +++ /dev/null @@ -1,359 +0,0 @@ -use std::collections::HashMap; - -use chrono::{DateTime, Duration, NaiveDateTime, Utc}; -use hmac::Mac; -use hyper::{Body, Method, Request}; -use sha2::{Digest, Sha256}; - -use garage_table::*; -use garage_util::data::Hash; - -use garage_model::garage::Garage; -use garage_model::key_table::*; - -use super::LONG_DATETIME; -use super::{compute_scope, signing_hmac}; - -use crate::encoding::uri_encode; -use crate::signature::error::*; - -pub async fn check_payload_signature( - garage: &Garage, - service: &'static str, - request: &Request, -) -> Result<(Option, Option), Error> { - let mut headers = HashMap::new(); - for (key, val) in request.headers() { - headers.insert(key.to_string(), val.to_str()?.to_string()); - } - if let Some(query) = request.uri().query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - for (key, val) in query_pairs { - headers.insert(key.to_lowercase(), val.to_string()); - } - } - - let authorization = if let Some(authorization) = headers.get("authorization") { - parse_authorization(authorization, &headers)? - } else if let Some(algorithm) = headers.get("x-amz-algorithm") { - parse_query_authorization(algorithm, &headers)? - } else { - let content_sha256 = headers.get("x-amz-content-sha256"); - if let Some(content_sha256) = content_sha256.filter(|c| "UNSIGNED-PAYLOAD" != c.as_str()) { - let sha256 = hex::decode(content_sha256) - .ok() - .and_then(|bytes| Hash::try_from(&bytes)) - .ok_or_bad_request("Invalid content sha256 hash")?; - return Ok((None, Some(sha256))); - } else { - return Ok((None, None)); - } - }; - - let canonical_request = canonical_request( - service, - request.method(), - request.uri(), - &headers, - &authorization.signed_headers, - &authorization.content_sha256, - ); - let (_, scope) = parse_credential(&authorization.credential)?; - let string_to_sign = string_to_sign(&authorization.date, &scope, &canonical_request); - - trace!("canonical request:\n{}", canonical_request); - trace!("string to sign:\n{}", string_to_sign); - - let key = verify_v4( - garage, - service, - &authorization.credential, - &authorization.date, - &authorization.signature, - string_to_sign.as_bytes(), - ) - .await?; - - let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" { - None - } else if authorization.content_sha256 == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" { - let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?) - } else { - let bytes = hex::decode(authorization.content_sha256) - .ok_or_bad_request("Invalid content sha256 hash")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?) - }; - - Ok((Some(key), content_sha256)) -} - -struct Authorization { - credential: String, - signed_headers: String, - signature: String, - content_sha256: String, - date: DateTime, -} - -fn parse_authorization( - authorization: &str, - headers: &HashMap, -) -> Result { - let first_space = authorization - .find(' ') - .ok_or_bad_request("Authorization field to short")?; - let (auth_kind, rest) = authorization.split_at(first_space); - - if auth_kind != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request("Unsupported authorization method")); - } - - let mut auth_params = HashMap::new(); - for auth_part in rest.split(',') { - let auth_part = auth_part.trim(); - let eq = auth_part - .find('=') - .ok_or_bad_request("Field without value in authorization header")?; - let (key, value) = auth_part.split_at(eq); - auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); - } - - let cred = auth_params - .get("Credential") - .ok_or_bad_request("Could not find Credential in Authorization field")?; - - let content_sha256 = headers - .get("x-amz-content-sha256") - .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; - - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; - - if Utc::now() - date > Duration::hours(24) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - let auth = Authorization { - credential: cred.to_string(), - signed_headers: auth_params - .get("SignedHeaders") - .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? - .to_string(), - signature: auth_params - .get("Signature") - .ok_or_bad_request("Could not find Signature in Authorization field")? - .to_string(), - content_sha256: content_sha256.to_string(), - date, - }; - Ok(auth) -} - -fn parse_query_authorization( - algorithm: &str, - headers: &HashMap, -) -> Result { - if algorithm != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request( - "Unsupported authorization method".to_string(), - )); - } - - let cred = headers - .get("x-amz-credential") - .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; - let signed_headers = headers - .get("x-amz-signedheaders") - .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; - let signature = headers - .get("x-amz-signature") - .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; - let content_sha256 = headers - .get("x-amz-content-sha256") - .map(|x| x.as_str()) - .unwrap_or("UNSIGNED-PAYLOAD"); - - let duration = headers - .get("x-amz-expires") - .ok_or_bad_request("X-Amz-Expires not found in query parameters")? - .parse() - .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; - - if duration > 7 * 24 * 3600 { - return Err(Error::bad_request( - "X-Amz-Expires may not exceed a week".to_string(), - )); - } - - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; - - if Utc::now() - date > Duration::seconds(duration) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - Ok(Authorization { - credential: cred.to_string(), - signed_headers: signed_headers.to_string(), - signature: signature.to_string(), - content_sha256: content_sha256.to_string(), - date, - }) -} - -fn parse_credential(cred: &str) -> Result<(String, String), Error> { - let first_slash = cred - .find('/') - .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; - let (key_id, scope) = cred.split_at(first_slash); - Ok(( - key_id.to_string(), - scope.trim_start_matches('/').to_string(), - )) -} - -pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { - let mut hasher = Sha256::default(); - hasher.update(canonical_req.as_bytes()); - [ - "AWS4-HMAC-SHA256", - &datetime.format(LONG_DATETIME).to_string(), - scope_string, - &hex::encode(hasher.finalize().as_slice()), - ] - .join("\n") -} - -pub fn canonical_request( - service: &'static str, - method: &Method, - uri: &hyper::Uri, - headers: &HashMap, - signed_headers: &str, - content_sha256: &str, -) -> String { - // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded - // a second time when building the canonical request, as specified in this documentation page: - // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html - // However this documentation page is for a specific service ("roles anywhere"), and - // in the S3 service we know for a fact that there is no double-urlencoding, because all of - // the tests we made with external software work without it. - // - // The theory is that double-urlencoding occurs for all services except S3, - // which is what is implemented in rusoto_signature: - // -> https://docs.rs/rusoto_signature/latest/src/rusoto_signature/signature.rs.html#464 - // - // Digging into the code of the official AWS Rust SDK, we learn that double-URI-encoding can - // be set or unset on a per-request basis (the signature crates, aws-sigv4 and aws-sig-auth, - // are agnostic to this). Grepping the codebase confirms that S3 is the only API for which - // double_uri_encode is set to false, meaning it is true (its default value) for all other - // AWS services. We will therefore implement this behavior in Garage as well. - // - // Note that this documentation page, which is touted as the "authoritative reference" on - // AWSv4 signatures, makes no mention of either single- or double-urlencoding: - // -> https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html - // This page of the S3 documentation does also not mention anything specific: - // -> https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - // - // Note that there is also the issue of path normalization, which I hope is unrelated to the - // one of URI-encoding. At least in aws-sigv4 both parameters can be set independently, - // and rusoto_signature does not seem to do any effective path normalization, even though - // it mentions it in the comments (same link to the souce code as above). - // We make the explicit choice of NOT normalizing paths in the K2V API because doing so - // would make non-normalized paths invalid K2V partition keys, and we don't want that. - let path: std::borrow::Cow = if service != "s3" { - uri_encode(uri.path(), false).into() - } else { - uri.path().into() - }; - [ - method.as_str(), - &path, - &canonical_query_string(uri), - &canonical_header_string(headers, signed_headers), - "", - signed_headers, - content_sha256, - ] - .join("\n") -} - -fn canonical_header_string(headers: &HashMap, signed_headers: &str) -> String { - let signed_headers_vec = signed_headers.split(';').collect::>(); - let mut items = headers - .iter() - .filter(|(key, _)| signed_headers_vec.contains(&key.as_str())) - .collect::>(); - items.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); - items - .iter() - .map(|(key, value)| key.to_lowercase() + ":" + value.trim()) - .collect::>() - .join("\n") -} - -fn canonical_query_string(uri: &hyper::Uri) -> String { - if let Some(query) = uri.query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - let mut items = query_pairs - .filter(|(key, _)| key != "X-Amz-Signature") - .map(|(key, value)| uri_encode(&key, true) + "=" + &uri_encode(&value, true)) - .collect::>(); - items.sort(); - items.join("&") - } else { - "".to_string() - } -} - -pub fn parse_date(date: &str) -> Result, Error> { - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - Ok(DateTime::from_utc(date, Utc)) -} - -pub async fn verify_v4( - garage: &Garage, - service: &str, - credential: &str, - date: &DateTime, - signature: &str, - payload: &[u8], -) -> Result { - let (key_id, scope) = parse_credential(credential)?; - - let scope_expected = compute_scope(date, &garage.config.s3_api.s3_region, service); - if scope != scope_expected { - return Err(Error::AuthorizationHeaderMalformed(scope.to_string())); - } - - let key = garage - .key_table - .get(&EmptyKey, &key_id) - .await? - .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::forbidden(format!("No such key: {}", &key_id)))?; - let key_p = key.params().unwrap(); - - let mut hmac = signing_hmac( - date, - &key_p.secret_key, - &garage.config.s3_api.s3_region, - service, - ) - .ok_or_internal_error("Unable to build signing HMAC")?; - hmac.update(payload); - let our_signature = hex::encode(hmac.finalize().into_bytes()); - if signature != our_signature { - return Err(Error::forbidden("Invalid signature".to_string())); - } - - Ok(key) -} diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs deleted file mode 100644 index c8358c4f..00000000 --- a/src/api/signature/streaming.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::pin::Pin; - -use chrono::{DateTime, NaiveDateTime, Utc}; -use futures::prelude::*; -use futures::task; -use garage_model::key_table::Key; -use hmac::Mac; -use hyper::body::Bytes; -use hyper::{Body, Request}; - -use garage_util::data::Hash; - -use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; - -use crate::signature::error::*; - -pub fn parse_streaming_body( - api_key: &Key, - req: Request, - content_sha256: &mut Option, - region: &str, - service: &str, -) -> Result, Error> { - match req.headers().get("x-amz-content-sha256") { - Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => { - let signature = content_sha256 - .take() - .ok_or_bad_request("No signature provided")?; - - let secret_key = &api_key - .state - .as_option() - .ok_or_internal_error("Deleted key state")? - .secret_key; - - let date = req - .headers() - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field")? - .to_str()?; - let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) - .ok_or_bad_request("Invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); - - let scope = compute_scope(&date, region, service); - let signing_hmac = crate::signature::signing_hmac(&date, secret_key, region, service) - .ok_or_internal_error("Unable to build signing HMAC")?; - - Ok(req.map(move |body| { - Body::wrap_stream( - SignedPayloadStream::new( - body.map_err(Error::from), - signing_hmac, - date, - &scope, - signature, - ) - .map_err(Error::from), - ) - })) - } - _ => Ok(req), - } -} - -/// Result of `sha256("")` -const EMPTY_STRING_HEX_DIGEST: &str = - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - -fn compute_streaming_payload_signature( - signing_hmac: &HmacSha256, - date: DateTime, - scope: &str, - previous_signature: Hash, - content_sha256: Hash, -) -> Result { - let string_to_sign = [ - "AWS4-HMAC-SHA256-PAYLOAD", - &date.format(LONG_DATETIME).to_string(), - scope, - &hex::encode(previous_signature), - EMPTY_STRING_HEX_DIGEST, - &hex::encode(content_sha256), - ] - .join("\n"); - - let mut hmac = signing_hmac.clone(); - hmac.update(string_to_sign.as_bytes()); - - Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?) -} - -mod payload { - use garage_util::data::Hash; - - pub enum Error { - Parser(nom::error::Error), - BadSignature, - } - - impl Error { - pub fn description(&self) -> &str { - match *self { - Error::Parser(ref e) => e.code.description(), - Error::BadSignature => "Bad signature", - } - } - } - - #[derive(Debug, Clone)] - pub struct Header { - pub size: usize, - pub signature: Hash, - } - - impl Header { - pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { - use nom::bytes::streaming::tag; - use nom::character::streaming::hex_digit1; - use nom::combinator::map_res; - use nom::number::streaming::hex_u32; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(|e| e.map(Error::Parser))? - }; - } - - let (input, size) = try_parse!(hex_u32(input)); - let (input, _) = try_parse!(tag(";")(input)); - - let (input, _) = try_parse!(tag("chunk-signature=")(input)); - let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); - let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; - - let (input, _) = try_parse!(tag("\r\n")(input)); - - let header = Header { - size: size as usize, - signature, - }; - - Ok((input, header)) - } - } -} - -#[derive(Debug)] -pub enum SignedPayloadStreamError { - Stream(Error), - InvalidSignature, - Message(String), -} - -impl SignedPayloadStreamError { - fn message(msg: &str) -> Self { - SignedPayloadStreamError::Message(msg.into()) - } -} - -impl From for Error { - fn from(err: SignedPayloadStreamError) -> Self { - match err { - SignedPayloadStreamError::Stream(e) => e, - SignedPayloadStreamError::InvalidSignature => { - Error::bad_request("Invalid payload signature") - } - SignedPayloadStreamError::Message(e) => { - Error::bad_request(format!("Chunk format error: {}", e)) - } - } - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: payload::Error) -> Self { - Self::message(err.description()) - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: nom::error::Error) -> Self { - Self::message(err.code.description()) - } -} - -struct SignedPayload { - header: payload::Header, - data: Bytes, -} - -#[pin_project::pin_project] -pub struct SignedPayloadStream -where - S: Stream>, -{ - #[pin] - stream: S, - buf: bytes::BytesMut, - datetime: DateTime, - scope: String, - signing_hmac: HmacSha256, - previous_signature: Hash, -} - -impl SignedPayloadStream -where - S: Stream>, -{ - pub fn new( - stream: S, - signing_hmac: HmacSha256, - datetime: DateTime, - scope: &str, - seed_signature: Hash, - ) -> Self { - Self { - stream, - buf: bytes::BytesMut::new(), - datetime, - scope: scope.into(), - signing_hmac, - previous_signature: seed_signature, - } - } - - fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> { - use nom::bytes::streaming::{tag, take}; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(nom::Err::convert)? - }; - } - - let (input, header) = try_parse!(payload::Header::parse(input)); - - // 0-sized chunk is the last - if header.size == 0 { - return Ok(( - input, - SignedPayload { - header, - data: Bytes::new(), - }, - )); - } - - let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); - let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); - - let data = Bytes::from(data.to_vec()); - - Ok((input, SignedPayload { header, data })) - } -} - -impl Stream for SignedPayloadStream -where - S: Stream> + Unpin, -{ - type Item = Result; - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut task::Context<'_>, - ) -> task::Poll> { - use std::task::Poll; - - let mut this = self.project(); - - loop { - let (input, payload) = match Self::parse_next(this.buf) { - Ok(res) => res, - Err(nom::Err::Incomplete(_)) => { - match futures::ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(bytes)) => { - this.buf.extend(bytes); - continue; - } - Some(Err(e)) => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e)))) - } - None => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::message( - "Unexpected EOF", - )))); - } - } - } - Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { - return Poll::Ready(Some(Err(e))) - } - }; - - // 0-sized chunk is the last - if payload.data.is_empty() { - return Poll::Ready(None); - } - - let data_sha256sum = sha256sum(&payload.data); - - let expected_signature = compute_streaming_payload_signature( - this.signing_hmac, - *this.datetime, - this.scope, - *this.previous_signature, - data_sha256sum, - ) - .map_err(|e| { - SignedPayloadStreamError::Message(format!("Could not build signature: {}", e)) - })?; - - if payload.header.signature != expected_signature { - return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature))); - } - - *this.buf = input.into(); - *this.previous_signature = payload.header.signature; - - return Poll::Ready(Some(Ok(payload.data))); - } - } - - fn size_hint(&self) -> (usize, Option) { - self.stream.size_hint() - } -} - -#[cfg(test)] -mod tests { - use futures::prelude::*; - - use super::{SignedPayloadStream, SignedPayloadStreamError}; - - #[tokio::test] - async fn test_interrupted_signed_payload_stream() { - use chrono::{DateTime, Utc}; - - use garage_util::data::Hash; - - let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 - .unwrap() - .with_timezone(&Utc); - let secret_key = "test"; - let region = "test"; - let scope = crate::signature::compute_scope(&datetime, region, "s3"); - let signing_hmac = - crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); - - let data: &[&[u8]] = &[b"1"]; - let body = futures::stream::iter(data.iter().map(|block| Ok(block.as_ref().into()))); - - let seed_signature = Hash::default(); - - let mut stream = - SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature); - - assert!(stream.try_next().await.is_err()); - match stream.try_next().await { - Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {} - item => panic!( - "Unexpected result, expected early EOF error, got {:?}", - item - ), - } - } -} diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 6a8f1af4..c4dbba44 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -15,30 +15,29 @@ path = "lib.rs" [dependencies] garage_db.workspace = true +garage_net.workspace = true garage_rpc.workspace = true garage_util.workspace = true garage_table.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true -arc-swap = "1.5" -async-trait = "0.1.7" -bytes = "1.0" -bytesize = "1.2" -hex = "0.4" -tracing = "0.1" -rand = "0.8" +arc-swap.workspace = true +async-trait.workspace = true +bytes.workspace = true +bytesize.workspace = true +hex.workspace = true +tracing.workspace = true +rand.workspace = true -async-compression = { version = "0.4", features = ["tokio", "zstd"] } -zstd = { version = "0.12", default-features = false } +async-compression.workspace = true +zstd.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-util = { version = "0.7", features = ["io"] } +futures.workspace = true +tokio.workspace = true +tokio-util.workspace = true [features] system-libs = [ "zstd/pkg-config" ] diff --git a/src/block/block.rs b/src/block/block.rs index 20f57aa5..bd95680e 100644 --- a/src/block/block.rs +++ b/src/block/block.rs @@ -2,110 +2,101 @@ use std::path::PathBuf; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use zstd::stream::{decode_all as zstd_decode, Encoder}; +use zstd::stream::Encoder; use garage_util::data::*; use garage_util::error::*; +use garage_net::stream::ByteStream; + #[derive(Debug, Serialize, Deserialize, Copy, Clone)] pub enum DataBlockHeader { Plain, Compressed, } -/// A possibly compressed block of data -pub enum DataBlock { - /// Uncompressed data - Plain(Bytes), - /// Data compressed with zstd - Compressed(Bytes), -} - #[derive(Debug)] -pub enum DataBlockPath { - /// Uncompressed data fail - Plain(PathBuf), - /// Compressed data fail - Compressed(PathBuf), +pub struct DataBlockElem { + header: DataBlockHeader, + elem: T, } -impl DataBlock { - /// Query whether this block is compressed +/// A possibly compressed block of data +pub type DataBlock = DataBlockElem; + +/// A path to a possibly compressed block of data +pub type DataBlockPath = DataBlockElem; + +/// A stream of possibly compressed block data +pub type DataBlockStream = DataBlockElem; + +impl DataBlockHeader { pub fn is_compressed(&self) -> bool { - matches!(self, DataBlock::Compressed(_)) + matches!(self, DataBlockHeader::Compressed) + } +} + +impl DataBlockElem { + pub fn from_parts(header: DataBlockHeader, elem: T) -> Self { + Self { header, elem } } - /// Get the inner, possibly compressed buffer. You should probably use [`DataBlock::verify_get`] - /// instead - pub fn inner_buffer(&self) -> &[u8] { - use DataBlock::*; - let (Plain(ref res) | Compressed(ref res)) = self; - res - } - - /// Get the buffer, possibly decompressing it, and verify it's integrity. - /// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system - /// is used instead. - pub fn verify_get(self, hash: Hash) -> Result { - match self { - DataBlock::Plain(data) => { - if blake2sum(&data) == hash { - Ok(data) - } else { - Err(Error::CorruptData(hash)) - } - } - DataBlock::Compressed(data) => zstd_decode(&data[..]) - .map_err(|_| Error::CorruptData(hash)) - .map(Bytes::from), + pub fn plain(elem: T) -> Self { + Self { + header: DataBlockHeader::Plain, + elem, } } - /// Verify data integrity. Allocate less than [`DataBlock::verify_get`] and don't consume self, but - /// does not return the buffer content. + pub fn compressed(elem: T) -> Self { + Self { + header: DataBlockHeader::Compressed, + elem, + } + } + + pub fn into_parts(self) -> (DataBlockHeader, T) { + (self.header, self.elem) + } + + pub fn as_parts_ref(&self) -> (DataBlockHeader, &T) { + (self.header, &self.elem) + } +} + +impl DataBlock { + /// Verify data integrity. Does not return the buffer content. pub fn verify(&self, hash: Hash) -> Result<(), Error> { - match self { - DataBlock::Plain(data) => { - if blake2sum(data) == hash { + match self.header { + DataBlockHeader::Plain => { + if blake2sum(&self.elem) == hash { Ok(()) } else { Err(Error::CorruptData(hash)) } } - DataBlock::Compressed(data) => zstd::stream::copy_decode(&data[..], std::io::sink()) - .map_err(|_| Error::CorruptData(hash)), + DataBlockHeader::Compressed => { + zstd::stream::copy_decode(&self.elem[..], std::io::sink()) + .map_err(|_| Error::CorruptData(hash)) + } } } pub async fn from_buffer(data: Bytes, level: Option) -> DataBlock { tokio::task::spawn_blocking(move || { if let Some(level) = level { - if let Ok(data) = zstd_encode(&data[..], level) { - return DataBlock::Compressed(data.into()); + if let Ok(data_compressed) = zstd_encode(&data[..], level) { + return DataBlock::compressed(data_compressed.into()); } } - DataBlock::Plain(data) + DataBlock::plain(data.into()) }) .await .unwrap() } - - pub fn into_parts(self) -> (DataBlockHeader, Bytes) { - match self { - DataBlock::Plain(data) => (DataBlockHeader::Plain, data), - DataBlock::Compressed(data) => (DataBlockHeader::Compressed, data), - } - } - - pub fn from_parts(h: DataBlockHeader, bytes: Bytes) -> Self { - match h { - DataBlockHeader::Plain => DataBlock::Plain(bytes), - DataBlockHeader::Compressed => DataBlock::Compressed(bytes), - } - } } -fn zstd_encode(mut source: R, level: i32) -> std::io::Result> { +pub fn zstd_encode(mut source: R, level: i32) -> std::io::Result> { let mut result = Vec::::new(); let mut encoder = Encoder::new(&mut result, level)?; encoder.include_checksum(true)?; diff --git a/src/block/layout.rs b/src/block/layout.rs index e8339405..00e3debb 100644 --- a/src/block/layout.rs +++ b/src/block/layout.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -13,9 +14,12 @@ const DRIVE_NPART: usize = 1024; const HASH_DRIVE_BYTES: (usize, usize) = (2, 3); +const MARKER_FILE_NAME: &str = "garage-marker"; + #[derive(Serialize, Deserialize, Debug, Clone)] pub(crate) struct DataLayout { pub(crate) data_dirs: Vec, + markers: HashMap, /// Primary storage location (index in data_dirs) for each partition /// = the location where the data is supposed to be, blocks are always @@ -75,16 +79,17 @@ impl DataLayout { Ok(Self { data_dirs, + markers: HashMap::new(), part_prim, part_sec, }) } - pub(crate) fn update(&mut self, dirs: &DataDirEnum) -> Result<(), Error> { + pub(crate) fn update(self, dirs: &DataDirEnum) -> Result { // Make list of new data directories, exit if nothing changed let data_dirs = make_data_dirs(dirs)?; if data_dirs == self.data_dirs { - return Ok(()); + return Ok(self); } let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::(); @@ -214,11 +219,43 @@ impl DataLayout { } // Apply newly generated config - *self = Self { + Ok(Self { data_dirs, + markers: self.markers, part_prim, part_sec, - }; + }) + } + + pub(crate) fn check_markers(&mut self) -> Result<(), Error> { + let data_dirs = &self.data_dirs; + self.markers + .retain(|k, _| data_dirs.iter().any(|x| x.path == *k)); + + for dir in self.data_dirs.iter() { + let mut marker_path = dir.path.clone(); + marker_path.push(MARKER_FILE_NAME); + let existing_marker = std::fs::read_to_string(&marker_path).ok(); + match (existing_marker, self.markers.get(&dir.path)) { + (Some(m1), Some(m2)) => { + if m1 != *m2 { + return Err(Error::Message(format!("Mismatched content for marker file `{}` in data directory `{}`. If you moved data directories or changed their mountpoints, you should remove the `data_layout` file in Garage's metadata directory and restart Garage.", MARKER_FILE_NAME, dir.path.display()))); + } + } + (None, Some(_)) => { + return Err(Error::Message(format!("Could not find expected marker file `{}` in data directory `{}`, make sure this data directory is mounted correctly.", MARKER_FILE_NAME, dir.path.display()))); + } + (Some(mkr), None) => { + self.markers.insert(dir.path.clone(), mkr); + } + (None, None) => { + let mkr = hex::encode(garage_util::data::gen_uuid().as_slice()); + std::fs::write(&marker_path, &mkr)?; + self.markers.insert(dir.path.clone(), mkr); + } + } + } + Ok(()) } @@ -242,7 +279,8 @@ impl DataLayout { u16::from_be_bytes([ hash.as_slice()[HASH_DRIVE_BYTES.0], hash.as_slice()[HASH_DRIVE_BYTES.1], - ]) as usize % DRIVE_NPART + ]) as usize + % DRIVE_NPART } fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf { @@ -255,6 +293,7 @@ impl DataLayout { pub(crate) fn without_secondary_locations(&self) -> Self { Self { data_dirs: self.data_dirs.clone(), + markers: self.markers.clone(), part_prim: self.part_prim.clone(), part_sec: self.part_sec.iter().map(|_| vec![]).collect::>(), } @@ -322,14 +361,12 @@ fn make_data_dirs(dirs: &DataDirEnum) -> Result, Error> { fn dir_not_empty(path: &PathBuf) -> Result { for entry in std::fs::read_dir(&path)? { let dir = entry?; - if dir.file_type()?.is_dir() - && dir - .file_name() - .into_string() - .ok() - .and_then(|hex| hex::decode(&hex).ok()) - .is_some() - { + let ft = dir.file_type()?; + let name = dir.file_name().into_string().ok(); + if ft.is_file() && name.as_deref() == Some(MARKER_FILE_NAME) { + return Ok(true); + } + if ft.is_dir() && name.and_then(|hex| hex::decode(&hex).ok()).is_some() { return Ok(true); } } diff --git a/src/block/lib.rs b/src/block/lib.rs index c9ff2845..944f0d83 100644 --- a/src/block/lib.rs +++ b/src/block/lib.rs @@ -9,3 +9,6 @@ mod block; mod layout; mod metrics; mod rc; + +pub use block::zstd_encode; +pub use rc::CalculateRefcount; diff --git a/src/block/manager.rs b/src/block/manager.rs index 2d1b5c67..96ca9c90 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -1,31 +1,28 @@ +use std::convert::TryInto; use std::path::PathBuf; -use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use arc_swap::{ArcSwap, ArcSwapOption}; -use async_trait::async_trait; use bytes::Bytes; use rand::prelude::*; use serde::{Deserialize, Serialize}; -use futures::Stream; -use futures_util::stream::StreamExt; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::sync::{mpsc, Mutex, MutexGuard}; +use tokio::sync::{mpsc, Mutex, MutexGuard, Semaphore}; use opentelemetry::{ trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, Context, }; -use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream}; +use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream}; use garage_db as db; use garage_util::background::{vars, BackgroundRunner}; -use garage_util::config::DataDirEnum; +use garage_util::config::Config; use garage_util::data::*; use garage_util::error::*; use garage_util::metrics::RecordDuration; @@ -53,6 +50,8 @@ pub const INLINE_THRESHOLD: usize = 3072; // to delete the block locally. pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600); +const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15); + /// RPC messages used to share blocks of data between nodes #[derive(Debug, Serialize, Deserialize)] pub enum BlockRpc { @@ -87,14 +86,17 @@ pub struct BlockManager { data_fsync: bool, compression_level: Option, + disable_scrub: bool, mutation_lock: Vec>, + read_semaphore: Semaphore, - pub(crate) rc: BlockRc, + pub rc: BlockRc, pub resync: BlockResyncManager, pub(crate) system: Arc, pub(crate) endpoint: Arc>, + buffer_kb_semaphore: Arc, pub(crate) metrics: BlockManagerMetrics, @@ -122,24 +124,22 @@ struct BlockManagerLocked(); impl BlockManager { pub fn new( db: &db::Db, - data_dir: DataDirEnum, - data_fsync: bool, - compression_level: Option, + config: &Config, replication: TableShardedReplication, system: Arc, ) -> Result, Error> { // Load or compute layout, i.e. assignment of data blocks to the different data directories let data_layout_persister: Persister = Persister::new(&system.metadata_dir, "data_layout"); - let data_layout = match data_layout_persister.load() { - Ok(mut layout) => { - layout - .update(&data_dir) - .ok_or_message("invalid data_dir config")?; - layout + let mut data_layout = match data_layout_persister.load() { + Ok(layout) => layout + .update(&config.data_dir) + .ok_or_message("invalid data_dir config")?, + Err(_) => { + DataLayout::initialize(&config.data_dir).ok_or_message("invalid data_dir config")? } - Err(_) => DataLayout::initialize(&data_dir).ok_or_message("invalid data_dir config")?, }; + data_layout.check_markers()?; data_layout_persister .save(&data_layout) .expect("cannot save data_layout"); @@ -156,11 +156,14 @@ impl BlockManager { .netapp .endpoint("garage_block/manager.rs/Rpc".to_string()); + let buffer_kb_semaphore = Arc::new(Semaphore::new(config.block_ram_buffer_max / 1024)); + let metrics = BlockManagerMetrics::new( - compression_level, - rc.rc.clone(), + config.compression_level, + rc.rc_table.clone(), resync.queue.clone(), resync.errors.clone(), + buffer_kb_semaphore.clone(), ); let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info"); @@ -169,16 +172,20 @@ impl BlockManager { replication, data_layout: ArcSwap::new(Arc::new(data_layout)), data_layout_persister, - data_fsync, - compression_level, + data_fsync: config.data_fsync, + disable_scrub: config.disable_scrub, + compression_level: config.compression_level, mutation_lock: vec![(); MUTEX_COUNT] .iter() .map(|_| Mutex::new(BlockManagerLocked())) .collect::>(), + + read_semaphore: Semaphore::new(config.block_max_concurrent_reads), rc, resync, system, endpoint, + buffer_kb_semaphore, metrics, scrub_persister, tx_scrub_command: ArcSwapOption::new(None), @@ -197,33 +204,43 @@ impl BlockManager { } // Spawn scrub worker - let (scrub_tx, scrub_rx) = mpsc::channel(1); - self.tx_scrub_command.store(Some(Arc::new(scrub_tx))); - bg.spawn_worker(ScrubWorker::new( - self.clone(), - scrub_rx, - self.scrub_persister.clone(), - )); + if !self.disable_scrub { + let (scrub_tx, scrub_rx) = mpsc::channel(1); + self.tx_scrub_command.store(Some(Arc::new(scrub_tx))); + bg.spawn_worker(ScrubWorker::new( + self.clone(), + scrub_rx, + self.scrub_persister.clone(), + )); + } } pub fn register_bg_vars(&self, vars: &mut vars::BgVars) { self.resync.register_bg_vars(vars); - vars.register_rw( - &self.scrub_persister, - "scrub-tranquility", - |p| p.get_with(|x| x.tranquility), - |p, tranquility| p.set_with(|x| x.tranquility = tranquility), - ); - vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| { - p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub)) - }); - vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| { - p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub)) - }); - vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| { - p.get_with(|x| x.corruptions_detected) - }); + if !self.disable_scrub { + vars.register_rw( + &self.scrub_persister, + "scrub-tranquility", + |p| p.get_with(|x| x.tranquility), + |p, tranquility| p.set_with(|x| x.tranquility = tranquility), + ); + vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| { + p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub)) + }); + vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| { + p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub)) + }); + vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| { + p.get_with(|x| x.corruptions_detected) + }); + } + } + + /// Initialization: set how block references are recalculated + /// for repair operations + pub fn set_recalc_rc(&self, recalc: Vec) { + self.rc.recalc_rc.store(Some(Arc::new(recalc))); } /// Ask nodes that might have a (possibly compressed) block for it @@ -231,11 +248,15 @@ impl BlockManager { async fn rpc_get_raw_block_streaming( &self, hash: &Hash, + priority: RequestPriority, order_tag: Option, - ) -> Result<(DataBlockHeader, ByteStream), Error> { - self.rpc_get_raw_block_internal(hash, order_tag, |header, stream| async move { - Ok((header, stream)) - }) + ) -> Result { + self.rpc_get_raw_block_internal( + hash, + priority, + order_tag, + |stream| async move { Ok(stream) }, + ) .await } @@ -244,12 +265,15 @@ impl BlockManager { pub(crate) async fn rpc_get_raw_block( &self, hash: &Hash, + priority: RequestPriority, order_tag: Option, ) -> Result { - self.rpc_get_raw_block_internal(hash, order_tag, |header, stream| async move { + self.rpc_get_raw_block_internal(hash, priority, order_tag, |block_stream| async move { + let (header, stream) = block_stream.into_parts(); read_stream_to_end(stream) .await - .map(|data| DataBlock::from_parts(header, data)) + .err_context("error in block data stream") + .map(|data| DataBlock::from_parts(header, data.into_bytes())) }) .await } @@ -257,22 +281,25 @@ impl BlockManager { async fn rpc_get_raw_block_internal( &self, hash: &Hash, + priority: RequestPriority, order_tag: Option, f: F, ) -> Result where - F: Fn(DataBlockHeader, ByteStream) -> Fut, + F: Fn(DataBlockStream) -> Fut, Fut: futures::Future>, { - let who = self.replication.read_nodes(hash); - let who = self.system.rpc.request_order(&who); + let who = self + .system + .rpc_helper() + .block_read_nodes_of(hash, self.system.rpc_helper()); for node in who.iter() { let node_id = NodeID::from(*node); let rpc = self.endpoint.call_streaming( &node_id, BlockRpc::GetBlock(*hash, order_tag), - PRIO_NORMAL | PRIO_SECONDARY, + priority, ); tokio::select! { res = rpc => { @@ -283,8 +310,8 @@ impl BlockManager { continue; } }; - let (header, stream) = match res.into_parts() { - (Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream), + let block_stream = match res.into_parts() { + (Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => DataBlockStream::from_parts(header, stream), (Ok(_), _) => { debug!("Get block {:?}: node {:?} returned a malformed response", hash, node); continue; @@ -294,7 +321,7 @@ impl BlockManager { continue; } }; - match f(header, stream).await { + match f(block_stream).await { Ok(ret) => return Ok(ret), Err(e) => { debug!("Get block {:?}: error reading stream from node {:?}: {}", hash, node, e); @@ -305,30 +332,29 @@ impl BlockManager { // if the first one doesn't succeed rapidly // TODO: keep first request running when initiating a new one and take the // one that finishes earlier - _ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => { + _ = tokio::time::sleep(self.system.rpc_helper().rpc_timeout()) => { debug!("Get block {:?}: node {:?} didn't return block in time, trying next.", hash, node); } }; } - let msg = format!("Get block {:?}: no node returned a valid block", hash); - debug!("{}", msg); - Err(Error::Message(msg)) + let err = Error::MissingBlock(*hash); + debug!("{}", err); + Err(err) } // ---- Public interface ---- - /// Ask nodes that might have a block for it, - /// return it as a stream + /// Ask nodes that might have a block for it, return it as a stream pub async fn rpc_get_block_streaming( &self, hash: &Hash, order_tag: Option, - ) -> Result< - Pin> + Send + Sync + 'static>>, - Error, - > { - let (header, stream) = self.rpc_get_raw_block_streaming(hash, order_tag).await?; + ) -> Result { + let block_stream = self + .rpc_get_raw_block_streaming(hash, PRIO_NORMAL | PRIO_SECONDARY, order_tag) + .await?; + let (header, stream) = block_stream.into_parts(); match header { DataBlockHeader::Plain => Ok(stream), DataBlockHeader::Compressed => { @@ -341,34 +367,44 @@ impl BlockManager { } } - /// Ask nodes that might have a block for it - pub async fn rpc_get_block( - &self, - hash: &Hash, - order_tag: Option, - ) -> Result { - self.rpc_get_raw_block(hash, order_tag) - .await? - .verify_get(*hash) - } - /// Send block to nodes that should have it - pub async fn rpc_put_block(&self, hash: Hash, data: Bytes) -> Result<(), Error> { - let who = self.replication.write_nodes(&hash); + pub async fn rpc_put_block( + &self, + hash: Hash, + data: Bytes, + prevent_compression: bool, + order_tag: Option, + ) -> Result<(), Error> { + let who = self.system.cluster_layout().current_storage_nodes_of(&hash); - let (header, bytes) = DataBlock::from_buffer(data, self.compression_level) + let compression_level = self.compression_level.filter(|_| !prevent_compression); + let (header, bytes) = DataBlock::from_buffer(data, compression_level) .await .into_parts(); + + let permit = self + .buffer_kb_semaphore + .clone() + .acquire_many_owned((bytes.len() / 1024).try_into().unwrap()) + .await + .ok_or_message("could not reserve space for buffer of data to send to remote nodes")?; + let put_block_rpc = Req::new(BlockRpc::PutBlock { hash, header })?.with_stream_from_buffer(bytes); + let put_block_rpc = if let Some(tag) = order_tag { + put_block_rpc.with_order_tag(tag) + } else { + put_block_rpc + }; self.system - .rpc - .try_call_many( + .rpc_helper() + .try_write_many_sets( &self.endpoint, - &who[..], + &[who], put_block_rpc, RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY) + .with_drop_on_completion(permit) .with_quorum(self.replication.write_quorum()), ) .await?; @@ -377,13 +413,8 @@ impl BlockManager { } /// Get number of items in the refcount table - pub fn rc_len(&self) -> Result { - Ok(self.rc.rc.len()?) - } - - /// Get number of items in the refcount table - pub fn rc_fast_len(&self) -> Result, Error> { - Ok(self.rc.rc.fast_len()?) + pub fn rc_approximate_len(&self) -> Result { + Ok(self.rc.rc_table.approximate_len()?) } /// Send command to start/stop/manager scrub worker @@ -401,7 +432,7 @@ impl BlockManager { /// List all resync errors pub fn list_resync_errors(&self) -> Result, Error> { - let mut blocks = Vec::with_capacity(self.resync.errors.len()); + let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?); for ent in self.resync.errors.iter()? { let (hash, cnt) = ent?; let cnt = ErrorCounter::decode(&cnt); @@ -439,7 +470,7 @@ impl BlockManager { tokio::spawn(async move { if let Err(e) = this .resync - .put_to_resync(&hash, 2 * this.system.rpc.rpc_timeout()) + .put_to_resync(&hash, 2 * this.system.rpc_helper().rpc_timeout()) { error!("Block {:?} could not be put in resync queue: {}.", hash, e); } @@ -482,7 +513,7 @@ impl BlockManager { stream: Option, ) -> Result<(), Error> { let stream = stream.ok_or_message("missing stream")?; - let bytes = read_stream_to_end(stream).await?; + let bytes = read_stream_to_end(stream).await?.into_bytes(); let data = DataBlock::from_parts(header, bytes); self.write_block(&hash, &data).await } @@ -531,9 +562,6 @@ impl BlockManager { match self.find_block(hash).await { Some(p) => self.read_block_from(hash, &p).await, None => { - // Not found but maybe we should have had it ?? - self.resync - .put_to_resync(hash, 2 * self.system.rpc.rpc_timeout())?; return Err(Error::Message(format!( "block {:?} not found on node", hash @@ -553,9 +581,15 @@ impl BlockManager { hash: &Hash, block_path: &DataBlockPath, ) -> Result { - let (path, compressed) = match block_path { - DataBlockPath::Plain(p) => (p, false), - DataBlockPath::Compressed(p) => (p, true), + let (header, path) = block_path.as_parts_ref(); + + let permit = tokio::select! { + sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?, + _ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => { + self.metrics.block_read_semaphore_timeouts.add(1); + debug!("read block {:?}: read_semaphore acquire timeout", hash); + return Err(Error::Message("read block: read_semaphore acquire timeout".into())); + } }; let mut f = fs::File::open(&path).await?; @@ -564,11 +598,7 @@ impl BlockManager { self.metrics.bytes_read.add(data.len() as u64); drop(f); - let data = if compressed { - DataBlock::Compressed(data.into()) - } else { - DataBlock::Plain(data.into()) - }; + let data = DataBlock::from_parts(header, data.into()); if data.verify(*hash).is_err() { self.metrics.corruption_counter.add(1); @@ -586,6 +616,8 @@ impl BlockManager { return Err(Error::CorruptData(*hash)); } + drop(permit); + Ok(data) } @@ -621,20 +653,20 @@ impl BlockManager { // first and then a compressed one (as compression may have been // previously enabled). if fs::metadata(&path).await.is_ok() { - return Some(DataBlockPath::Plain(path)); + return Some(DataBlockPath::plain(path)); } path.set_extension("zst"); if fs::metadata(&path).await.is_ok() { - return Some(DataBlockPath::Compressed(path)); + return Some(DataBlockPath::compressed(path)); } } else { path.set_extension("zst"); if fs::metadata(&path).await.is_ok() { - return Some(DataBlockPath::Compressed(path)); + return Some(DataBlockPath::compressed(path)); } path.set_extension(""); if fs::metadata(&path).await.is_ok() { - return Some(DataBlockPath::Plain(path)); + return Some(DataBlockPath::plain(path)); } } } @@ -649,10 +681,12 @@ impl BlockManager { hash: &Hash, wrong_path: DataBlockPath, ) -> Result { + let data = self.read_block_from(hash, &wrong_path).await?; self.lock_mutate(hash) .await - .fix_block_location(hash, wrong_path, self) - .await + .write_block_inner(hash, &data, self, Some(wrong_path)) + .await?; + Ok(data.as_parts_ref().1.len()) } async fn lock_mutate(&self, hash: &Hash) -> MutexGuard<'_, BlockManagerLocked> { @@ -668,7 +702,6 @@ impl BlockManager { } } -#[async_trait] impl StreamingEndpointHandler for BlockManager { async fn handle(self: &Arc, mut message: Req, _from: NodeID) -> Resp { match message.msg() { @@ -704,8 +737,8 @@ impl BlockManagerLocked { mgr: &BlockManager, existing_path: Option, ) -> Result<(), Error> { - let compressed = data.is_compressed(); - let data = data.inner_buffer(); + let (header, data) = data.as_parts_ref(); + let compressed = header.is_compressed(); let directory = mgr.data_layout.load().primary_block_dir(hash); @@ -715,24 +748,25 @@ impl BlockManagerLocked { tgt_path.set_extension("zst"); } - let to_delete = match (existing_path, compressed) { + let existing_info = existing_path.map(|x| x.into_parts()); + let to_delete = match (existing_info, compressed) { // If the block is stored in the wrong directory, // write it again at the correct path and delete the old path - (Some(DataBlockPath::Plain(p)), false) if p != tgt_path => Some(p), - (Some(DataBlockPath::Compressed(p)), true) if p != tgt_path => Some(p), + (Some((DataBlockHeader::Plain, p)), false) if p != tgt_path => Some(p), + (Some((DataBlockHeader::Compressed, p)), true) if p != tgt_path => Some(p), // If the block is already stored not compressed but we have a compressed // copy, write the compressed copy and delete the uncompressed one - (Some(DataBlockPath::Plain(plain_path)), true) => Some(plain_path), + (Some((DataBlockHeader::Plain, plain_path)), true) => Some(plain_path), // If the block is already stored compressed, // keep the stored copy, we have nothing to do - (Some(DataBlockPath::Compressed(_)), _) => return Ok(()), + (Some((DataBlockHeader::Compressed, _)), _) => return Ok(()), // If the block is already stored not compressed, // and we don't have a compressed copy either, // keep the stored copy, we have nothing to do - (Some(DataBlockPath::Plain(_)), false) => return Ok(()), + (Some((DataBlockHeader::Plain, _)), false) => return Ok(()), // If the block isn't stored already, just store what is given to us (None, _) => None, @@ -749,6 +783,7 @@ impl BlockManagerLocked { let mut f = fs::File::create(&path_tmp).await?; f.write_all(data).await?; + f.flush().await?; mgr.metrics.bytes_written.add(data.len() as u64); if mgr.data_fsync { @@ -784,18 +819,14 @@ impl BlockManagerLocked { } async fn move_block_to_corrupted(&self, block_path: &DataBlockPath) -> Result<(), Error> { - let (path, path2) = match block_path { - DataBlockPath::Plain(p) => { - let mut p2 = p.clone(); - p2.set_extension("corrupted"); - (p, p2) - } - DataBlockPath::Compressed(p) => { - let mut p2 = p.clone(); - p2.set_extension("zst.corrupted"); - (p, p2) - } - }; + let (header, path) = block_path.as_parts_ref(); + + let mut path2 = path.clone(); + if header.is_compressed() { + path2.set_extension("zst.corrupted"); + } else { + path2.set_extension("corrupted"); + } fs::rename(path, path2).await?; Ok(()) @@ -805,41 +836,13 @@ impl BlockManagerLocked { let rc = mgr.rc.get_block_rc(hash)?; if rc.is_deletable() { while let Some(path) = mgr.find_block(hash).await { - let path = match path { - DataBlockPath::Plain(p) | DataBlockPath::Compressed(p) => p, - }; + let (_header, path) = path.as_parts_ref(); fs::remove_file(path).await?; mgr.metrics.delete_counter.add(1); } } Ok(()) } - - async fn fix_block_location( - &self, - hash: &Hash, - wrong_path: DataBlockPath, - mgr: &BlockManager, - ) -> Result { - let data = mgr.read_block_from(hash, &wrong_path).await?; - self.write_block_inner(hash, &data, mgr, Some(wrong_path)) - .await?; - Ok(data.inner_buffer().len()) - } -} - -async fn read_stream_to_end(mut stream: ByteStream) -> Result { - let mut parts: Vec = vec![]; - while let Some(part) = stream.next().await { - parts.push(part.ok_or_message("error in stream")?); - } - - Ok(parts - .iter() - .map(|x| &x[..]) - .collect::>() - .concat() - .into()) } struct DeleteOnDrop(Option); diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 6659df32..81021fe1 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,7 +1,10 @@ +use std::sync::Arc; + +use tokio::sync::Semaphore; + use opentelemetry::{global, metrics::*}; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { @@ -9,6 +12,7 @@ pub struct BlockManagerMetrics { pub(crate) _rc_size: ValueObserver, pub(crate) _resync_queue_len: ValueObserver, pub(crate) _resync_errored_blocks: ValueObserver, + pub(crate) _buffer_free_kb: ValueObserver, pub(crate) resync_counter: BoundCounter, pub(crate) resync_error_counter: BoundCounter, @@ -18,6 +22,7 @@ pub struct BlockManagerMetrics { pub(crate) bytes_read: BoundCounter, pub(crate) block_read_duration: BoundValueRecorder, + pub(crate) block_read_semaphore_timeouts: BoundCounter, pub(crate) bytes_written: BoundCounter, pub(crate) block_write_duration: BoundValueRecorder, pub(crate) delete_counter: BoundCounter, @@ -29,8 +34,9 @@ impl BlockManagerMetrics { pub fn new( compression_level: Option, rc_tree: db::Tree, - resync_queue: CountedTree, - resync_errors: CountedTree, + resync_queue: db::Tree, + resync_errors: db::Tree, + buffer_semaphore: Arc, ) -> Self { let meter = global::meter("garage_model/block"); Self { @@ -45,15 +51,17 @@ impl BlockManagerMetrics { .init(), _rc_size: meter .u64_value_observer("block.rc_size", move |observer| { - if let Ok(Some(v)) = rc_tree.fast_len() { - observer.observe(v as u64, &[]) + if let Ok(value) = rc_tree.approximate_len() { + observer.observe(value as u64, &[]) } }) .with_description("Number of blocks known to the reference counter") .init(), _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - observer.observe(resync_queue.len() as u64, &[]) + if let Ok(value) = resync_queue.approximate_len() { + observer.observe(value as u64, &[]); + } }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -61,11 +69,22 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - observer.observe(resync_errors.len() as u64, &[]) + if let Ok(value) = resync_errors.approximate_len() { + observer.observe(value as u64, &[]); + } }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), + _buffer_free_kb: meter + .u64_value_observer("block.ram_buffer_free_kb", move |observer| { + observer.observe(buffer_semaphore.available_permits() as u64, &[]) + }) + .with_description( + "Available RAM in KiB to use for buffering data blocks to be written to remote nodes", + ) + .init(), + resync_counter: meter .u64_counter("block.resync_counter") .with_description("Number of calls to resync_block") @@ -101,6 +120,11 @@ impl BlockManagerMetrics { .with_description("Duration of block read operations") .init() .bind(&[]), + block_read_semaphore_timeouts: meter + .u64_counter("block.read_semaphore_timeouts") + .with_description("Number of block reads that failed due to semaphore acquire timeout") + .init() + .bind(&[]), bytes_written: meter .u64_counter("block.bytes_written") .with_description("Number of bytes written to disk") diff --git a/src/block/rc.rs b/src/block/rc.rs index b6afb277..4a55ee29 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -1,5 +1,7 @@ use std::convert::TryInto; +use arc_swap::ArcSwapOption; + use garage_db as db; use garage_util::data::*; @@ -8,13 +10,20 @@ use garage_util::time::*; use crate::manager::BLOCK_GC_DELAY; +pub type CalculateRefcount = + Box db::TxResult + Send + Sync>; + pub struct BlockRc { - pub(crate) rc: db::Tree, + pub rc_table: db::Tree, + pub(crate) recalc_rc: ArcSwapOption>, } impl BlockRc { pub(crate) fn new(rc: db::Tree) -> Self { - Self { rc } + Self { + rc_table: rc, + recalc_rc: ArcSwapOption::new(None), + } } /// Increment the reference counter associated to a hash. @@ -24,9 +33,9 @@ impl BlockRc { tx: &mut db::Transaction, hash: &Hash, ) -> db::TxOpResult { - let old_rc = RcEntry::parse_opt(tx.get(&self.rc, hash)?); + let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); match old_rc.increment().serialize() { - Some(x) => tx.insert(&self.rc, hash, x)?, + Some(x) => tx.insert(&self.rc_table, hash, x)?, None => unreachable!(), }; Ok(old_rc.is_zero()) @@ -39,28 +48,28 @@ impl BlockRc { tx: &mut db::Transaction, hash: &Hash, ) -> db::TxOpResult { - let new_rc = RcEntry::parse_opt(tx.get(&self.rc, hash)?).decrement(); + let new_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?).decrement(); match new_rc.serialize() { - Some(x) => tx.insert(&self.rc, hash, x)?, - None => tx.remove(&self.rc, hash)?, + Some(x) => tx.insert(&self.rc_table, hash, x)?, + None => tx.remove(&self.rc_table, hash)?, }; Ok(matches!(new_rc, RcEntry::Deletable { .. })) } /// Read a block's reference count pub(crate) fn get_block_rc(&self, hash: &Hash) -> Result { - Ok(RcEntry::parse_opt(self.rc.get(hash.as_ref())?)) + Ok(RcEntry::parse_opt(self.rc_table.get(hash.as_ref())?)) } /// Delete an entry in the RC table if it is deletable and the /// deletion time has passed pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> { let now = now_msec(); - self.rc.db().transaction(|tx| { - let rcval = RcEntry::parse_opt(tx.get(&self.rc, hash)?); + self.rc_table.db().transaction(|tx| { + let rcval = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); match rcval { RcEntry::Deletable { at_time } if now > at_time => { - tx.remove(&self.rc, hash)?; + tx.remove(&self.rc_table, hash)?; } _ => (), }; @@ -68,6 +77,58 @@ impl BlockRc { })?; Ok(()) } + + /// Recalculate the reference counter of a block + /// to fix potential inconsistencies + pub fn recalculate_rc(&self, hash: &Hash) -> Result<(usize, bool), Error> { + if let Some(recalc_fns) = self.recalc_rc.load().as_ref() { + trace!("Repair block RC for {:?}", hash); + let res = self + .rc_table + .db() + .transaction(|tx| { + let mut cnt = 0; + for f in recalc_fns.iter() { + cnt += f(&tx, hash)?; + } + let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); + trace!( + "Block RC for {:?}: stored={}, calculated={}", + hash, + old_rc.as_u64(), + cnt + ); + if cnt as u64 != old_rc.as_u64() { + warn!( + "Fixing inconsistent block RC for {:?}: was {}, should be {}", + hash, + old_rc.as_u64(), + cnt + ); + let new_rc = if cnt > 0 { + RcEntry::Present { count: cnt as u64 } + } else { + RcEntry::Deletable { + at_time: now_msec() + BLOCK_GC_DELAY.as_millis() as u64, + } + }; + tx.insert(&self.rc_table, hash, new_rc.serialize().unwrap())?; + Ok((cnt, true)) + } else { + Ok((cnt, false)) + } + }) + .map_err(Error::from); + if let Err(e) = &res { + error!("Failed to fix RC for block {:?}: {}", hash, e); + } + res + } else { + Err(Error::Message( + "Block RC recalculation is not available at this point".into(), + )) + } + } } /// Describes the state of the reference counter for a block diff --git a/src/block/repair.rs b/src/block/repair.rs index 77ee0d14..ef271094 100644 --- a/src/block/repair.rs +++ b/src/block/repair.rs @@ -107,7 +107,7 @@ impl Worker for RepairWorker { for entry in self .manager .rc - .rc + .rc_table .range::<&[u8], _>((start_bound, Bound::Unbounded))? { let (hash, _) = entry?; @@ -584,8 +584,8 @@ impl Worker for RebalanceWorker { let prim_loc = self.manager.data_layout.load().primary_block_dir(&hash); if path.ancestors().all(|x| x != prim_loc) { let block_path = match path.extension() { - None => DataBlockPath::Plain(path.clone()), - Some(x) if x.to_str() == Some("zst") => DataBlockPath::Compressed(path.clone()), + None => DataBlockPath::plain(path.clone()), + Some(x) if x.to_str() == Some("zst") => DataBlockPath::compressed(path.clone()), _ => { warn!("not rebalancing file: {}", path.to_string_lossy()); return Ok(WorkerState::Busy); diff --git a/src/block/resync.rs b/src/block/resync.rs index 9c1da4a7..7056a828 100644 --- a/src/block/resync.rs +++ b/src/block/resync.rs @@ -15,7 +15,6 @@ use opentelemetry::{ }; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; use garage_util::background::*; use garage_util::data::*; @@ -47,9 +46,9 @@ pub(crate) const MAX_RESYNC_WORKERS: usize = 8; const INITIAL_RESYNC_TRANQUILITY: u32 = 2; pub struct BlockResyncManager { - pub(crate) queue: CountedTree, + pub(crate) queue: db::Tree, pub(crate) notify: Arc, - pub(crate) errors: CountedTree, + pub(crate) errors: db::Tree, busy_set: BusySet, @@ -90,12 +89,10 @@ impl BlockResyncManager { let queue = db .open_tree("block_local_resync_queue") .expect("Unable to open block_local_resync_queue tree"); - let queue = CountedTree::new(queue).expect("Could not count block_local_resync_queue"); let errors = db .open_tree("block_local_resync_errors") .expect("Unable to open block_local_resync_errors tree"); - let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors"); let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg"); @@ -108,18 +105,14 @@ impl BlockResyncManager { } } - /// Get lenght of resync queue - pub fn queue_len(&self) -> Result { - // This currently can't return an error because the CountedTree hack - // doesn't error on .len(), but this will change when we remove the hack - // (hopefully someday!) - Ok(self.queue.len()) + /// Get length of resync queue + pub fn queue_approximate_len(&self) -> Result { + Ok(self.queue.approximate_len()?) } /// Get number of blocks that have an error - pub fn errors_len(&self) -> Result { - // (see queue_len comment) - Ok(self.errors.len()) + pub fn errors_approximate_len(&self) -> Result { + Ok(self.errors.approximate_len()?) } /// Clear the error counter for a block and put it in queue immediately @@ -140,6 +133,14 @@ impl BlockResyncManager { ))) } + /// Clear the entire resync queue and list of errored blocks + /// Corresponds to `garage repair clear-resync-queue` + pub fn clear_resync_queue(&self) -> Result<(), Error> { + self.queue.clear()?; + self.errors.clear()?; + Ok(()) + } + pub fn register_bg_vars(&self, vars: &mut vars::BgVars) { let notify = self.notify.clone(); vars.register_rw( @@ -180,7 +181,7 @@ impl BlockResyncManager { // deleted once the garbage collection delay has passed. // // Here are some explanations on how the resync queue works. - // There are two Sled trees that are used to have information + // There are two db trees that are used to have information // about the status of blocks that need to be resynchronized: // // - resync.queue: a tree that is ordered first by a timestamp @@ -192,10 +193,10 @@ impl BlockResyncManager { // // - resync.errors: a tree that indicates for each block // if the last resync resulted in an error, and if so, - // the following two informations (see the ErrorCounter struct): + // the following two information (see the ErrorCounter struct): // - how many consecutive resync errors for this block? // - when was the last try? - // These two informations are used to implement an + // These two information are used to implement an // exponential backoff retry strategy. // The key in this tree is the 32-byte hash of the block, // and the value is the encoded ErrorCounter value. @@ -374,10 +375,20 @@ impl BlockResyncManager { } if exists && rc.is_deletable() { + if manager.rc.recalculate_rc(hash)?.0 > 0 { + return Err(Error::Message(format!( + "Refcount for block {:?} was inconsistent, retrying later", + hash + ))); + } + info!("Resync block {:?}: offloading and deleting", hash); let existing_path = existing_path.unwrap(); - let mut who = manager.replication.write_nodes(hash); + let mut who = manager + .system + .cluster_layout() + .current_storage_nodes_of(hash); if who.len() < manager.replication.write_quorum() { return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string())); } @@ -385,7 +396,7 @@ impl BlockResyncManager { let who_needs_resps = manager .system - .rpc + .rpc_helper() .call_many( &manager.endpoint, &who, @@ -431,12 +442,12 @@ impl BlockResyncManager { .with_stream_from_buffer(bytes); manager .system - .rpc + .rpc_helper() .try_call_many( &manager.endpoint, - &need_nodes[..], + &need_nodes, put_block_message, - RequestStrategy::with_priority(PRIO_BACKGROUND) + RequestStrategy::with_priority(PRIO_BACKGROUND | PRIO_SECONDARY) .with_quorum(need_nodes.len()), ) .await @@ -455,12 +466,41 @@ impl BlockResyncManager { } if rc.is_nonzero() && !exists { + // The refcount is > 0, and the block is not present locally. + // We might need to fetch it from another node. + + // First, check whether we are still supposed to store that + // block in the latest cluster layout version. + let storage_nodes = manager + .system + .cluster_layout() + .current_storage_nodes_of(&hash); + + if !storage_nodes.contains(&manager.system.id) { + info!( + "Resync block {:?}: block is absent with refcount > 0, but it will drop to zero after all metadata is synced. Not fetching the block.", + hash + ); + return Ok(()); + } + + // We know we need the block. Fetch it. info!( "Resync block {:?}: fetching absent but needed block (refcount > 0)", hash ); - let block_data = manager.rpc_get_raw_block(hash, None).await?; + let block_data = manager + .rpc_get_raw_block(hash, PRIO_BACKGROUND | PRIO_SECONDARY, None) + .await; + if matches!(block_data, Err(Error::MissingBlock(_))) { + warn!( + "Could not fetch needed block {:?}, no node returned valid data. Checking that refcount is correct.", + hash + ); + manager.rc.recalculate_rc(hash)?; + } + let block_data = block_data?; manager.metrics.resync_recv_counter.add(1); @@ -516,9 +556,11 @@ impl Worker for ResyncWorker { } WorkerStatus { - queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64), + queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64), tranquility: Some(tranquility), - persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64), + persistent_errors: Some( + self.manager.resync.errors_approximate_len().unwrap_or(0) as u64 + ), ..Default::default() } } @@ -541,9 +583,9 @@ impl Worker for ResyncWorker { Ok(WorkerState::Idle) } Err(e) => { - // The errors that we have here are only Sled errors + // The errors that we have here are only db errors // We don't really know how to handle them so just ¯\_(ツ)_/¯ - // (there is kind of an assumption that Sled won't error on us, + // (there is kind of an assumption that the db won't error on us, // if it does there is not much we can do -- TODO should we just panic?) // Here we just give the error to the worker manager, // it will print it to the logs and increment a counter diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index e479ed8f..9e860e7d 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -11,30 +11,25 @@ readme = "../../README.md" [lib] path = "lib.rs" -[[bin]] -name = "convert" -path = "bin/convert.rs" -required-features = ["cli"] - [dependencies] -err-derive = "0.3" -hexdump = "0.1" -tracing = "0.1" +thiserror.workspace = true +tracing.workspace = true -heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } -rusqlite = { version = "0.29", optional = true } -sled = { version = "0.34", optional = true } +heed = { workspace = true, optional = true } -# cli deps -clap = { version = "4.1", optional = true, features = ["derive", "env"] } -pretty_env_logger = { version = "0.5", optional = true } +rusqlite = { workspace = true, optional = true, features = ["backup"] } +r2d2 = { workspace = true, optional = true } +r2d2_sqlite = { workspace = true, optional = true } + +fjall = { workspace = true, optional = true } +parking_lot = { workspace = true, optional = true } [dev-dependencies] -mktemp = "0.5" +mktemp.workspace = true [features] -default = [ "sled", "lmdb", "sqlite" ] +default = [ "lmdb", "sqlite" ] bundled-libs = [ "rusqlite?/bundled" ] -cli = ["clap", "pretty_env_logger"] lmdb = [ "heed" ] -sqlite = [ "rusqlite" ] +fjall = [ "dep:fjall", "dep:parking_lot" ] +sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ] diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs deleted file mode 100644 index 957deedf..00000000 --- a/src/db/bin/convert.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::path::PathBuf; - -use garage_db::*; - -use clap::Parser; - -/// K2V command line interface -#[derive(Parser, Debug)] -#[clap(author, version, about, long_about = None)] -struct Args { - /// Input DB path - #[clap(short = 'i')] - input_path: PathBuf, - /// Input DB engine - #[clap(short = 'a')] - input_engine: String, - - /// Output DB path - #[clap(short = 'o')] - output_path: PathBuf, - /// Output DB engine - #[clap(short = 'b')] - output_engine: String, -} - -fn main() { - let args = Args::parse(); - pretty_env_logger::init(); - - match do_conversion(args) { - Ok(()) => println!("Success!"), - Err(e) => eprintln!("Error: {}", e), - } -} - -fn do_conversion(args: Args) -> Result<()> { - let input = open_db(args.input_path, args.input_engine)?; - let output = open_db(args.output_path, args.output_engine)?; - output.import(&input)?; - Ok(()) -} - -fn open_db(path: PathBuf, engine: String) -> Result { - match engine.as_str() { - "sled" => { - let db = sled_adapter::sled::Config::default().path(&path).open()?; - Ok(sled_adapter::SledDb::init(db)) - } - "sqlite" | "sqlite3" | "rusqlite" => { - let db = sqlite_adapter::rusqlite::Connection::open(&path)?; - db.pragma_update(None, "journal_mode", &"WAL")?; - db.pragma_update(None, "synchronous", &"NORMAL")?; - Ok(sqlite_adapter::SqliteDb::init(db)) - } - "lmdb" | "heed" => { - std::fs::create_dir_all(&path).map_err(|e| { - Error(format!("Unable to create LMDB data directory: {}", e).into()) - })?; - - let map_size = lmdb_adapter::recommended_map_size(); - - let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); - env_builder.max_dbs(100); - env_builder.map_size(map_size); - unsafe { - env_builder.flag(heed::flags::Flags::MdbNoMetaSync); - } - let db = env_builder.open(&path)?; - Ok(lmdb_adapter::LmdbDb::init(db)) - } - e => Err(Error(format!("Invalid DB engine: {}", e).into())), - } -} diff --git a/src/db/counted_tree_hack.rs b/src/db/counted_tree_hack.rs deleted file mode 100644 index a4ce12e0..00000000 --- a/src/db/counted_tree_hack.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! This hack allows a db tree to keep in RAM a counter of the number of entries -//! it contains, which is used to call .len() on it. This is usefull only for -//! the sled backend where .len() otherwise would have to traverse the whole -//! tree to count items. For sqlite and lmdb, this is mostly useless (but -//! hopefully not harmfull!). Note that a CountedTree cannot be part of a -//! transaction. - -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; - -use crate::{Result, Tree, TxError, Value, ValueIter}; - -#[derive(Clone)] -pub struct CountedTree(Arc); - -struct CountedTreeInternal { - tree: Tree, - len: AtomicUsize, -} - -impl CountedTree { - pub fn new(tree: Tree) -> Result { - let len = tree.len()?; - Ok(Self(Arc::new(CountedTreeInternal { - tree, - len: AtomicUsize::new(len), - }))) - } - - pub fn len(&self) -> usize { - self.0.len.load(Ordering::SeqCst) - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn get>(&self, key: K) -> Result> { - self.0.tree.get(key) - } - - pub fn first(&self) -> Result> { - self.0.tree.first() - } - - pub fn iter(&self) -> Result> { - self.0.tree.iter() - } - - // ---- writing functions ---- - - pub fn insert(&self, key: K, value: V) -> Result> - where - K: AsRef<[u8]>, - V: AsRef<[u8]>, - { - let old_val = self.0.tree.insert(key, value)?; - if old_val.is_none() { - self.0.len.fetch_add(1, Ordering::SeqCst); - } - Ok(old_val) - } - - pub fn remove>(&self, key: K) -> Result> { - let old_val = self.0.tree.remove(key)?; - if old_val.is_some() { - self.0.len.fetch_sub(1, Ordering::SeqCst); - } - Ok(old_val) - } - - pub fn compare_and_swap( - &self, - key: K, - expected_old: Option, - new: Option, - ) -> Result - where - K: AsRef<[u8]>, - OV: AsRef<[u8]>, - NV: AsRef<[u8]>, - { - let old_some = expected_old.is_some(); - let new_some = new.is_some(); - - let tx_res = self.0.tree.db().transaction(|tx| { - let old_val = tx.get(&self.0.tree, &key)?; - let is_same = match (&old_val, &expected_old) { - (None, None) => true, - (Some(x), Some(y)) if x == y.as_ref() => true, - _ => false, - }; - if is_same { - match &new { - Some(v) => { - tx.insert(&self.0.tree, &key, v)?; - } - None => { - tx.remove(&self.0.tree, &key)?; - } - } - Ok(()) - } else { - Err(TxError::Abort(())) - } - }); - - match tx_res { - Ok(()) => { - match (old_some, new_some) { - (false, true) => { - self.0.len.fetch_add(1, Ordering::SeqCst); - } - (true, false) => { - self.0.len.fetch_sub(1, Ordering::SeqCst); - } - _ => (), - } - Ok(true) - } - Err(TxError::Abort(())) => Ok(false), - Err(TxError::Db(e)) => Err(e), - } - } -} diff --git a/src/db/fjall_adapter.rs b/src/db/fjall_adapter.rs new file mode 100644 index 00000000..25913a1f --- /dev/null +++ b/src/db/fjall_adapter.rs @@ -0,0 +1,453 @@ +use core::ops::Bound; + +use std::path::PathBuf; +use std::sync::Arc; + +use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard}; + +use fjall::{ + PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle, + WriteTransaction, +}; + +use crate::{ + open::{Engine, OpenOpt}, + Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, + TxResult, TxValueIter, Value, ValueIter, +}; + +pub use fjall; + +// -- + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening Fjall database at: {}", path.display()); + if opt.fsync { + return Err(Error( + "metadata_fsync is not supported with the Fjall database engine".into(), + )); + } + let mut config = fjall::Config::new(path); + if let Some(block_cache_size) = opt.fjall_block_cache_size { + config = config.cache_size(block_cache_size as u64); + } + let keyspace = config.open_transactional()?; + Ok(FjallDb::init(keyspace)) +} + +// -- err + +impl From for Error { + fn from(e: fjall::Error) -> Error { + Error(format!("fjall: {}", e).into()) + } +} + +impl From for Error { + fn from(e: fjall::LsmError) -> Error { + Error(format!("fjall lsm_tree: {}", e).into()) + } +} + +impl From for TxOpError { + fn from(e: fjall::Error) -> TxOpError { + TxOpError(e.into()) + } +} + +// -- db + +pub struct FjallDb { + keyspace: TransactionalKeyspace, + trees: RwLock>, +} + +type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>); + +impl FjallDb { + pub fn init(keyspace: TransactionalKeyspace) -> Db { + let s = Self { + keyspace, + trees: RwLock::new(Vec::new()), + }; + Db(Arc::new(s)) + } + + fn get_tree( + &self, + i: usize, + ) -> Result> { + RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| { + trees.get(i).map(|tup| &tup.1) + }) + .map_err(|_| Error("invalid tree id".into())) + } +} + +impl IDb for FjallDb { + fn engine(&self) -> String { + "Fjall (EXPERIMENTAL!)".into() + } + + fn open_tree(&self, name: &str) -> Result { + let mut trees = self.trees.write(); + let safe_name = encode_name(name)?; + if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) { + Ok(i) + } else { + let tree = self + .keyspace + .open_partition(&safe_name, PartitionCreateOptions::default())?; + let i = trees.len(); + trees.push((safe_name, tree)); + Ok(i) + } + } + + fn list_trees(&self) -> Result> { + Ok(self + .keyspace + .list_partitions() + .iter() + .map(|n| decode_name(&n)) + .collect::>>()?) + } + + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Fjall.db_path(base_path); + + let source_state = self.keyspace.read_tx(); + let copy_keyspace = fjall::Config::new(path).open()?; + + for partition_name in self.keyspace.list_partitions() { + let source_partition = self + .keyspace + .open_partition(&partition_name, PartitionCreateOptions::default())?; + let copy_partition = + copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?; + + for entry in source_state.iter(&source_partition) { + let (key, value) = entry?; + copy_partition.insert(key, value)?; + } + } + + copy_keyspace.persist(PersistMode::SyncAll)?; + Ok(()) + } + + // ---- + + fn get(&self, tree_idx: usize, key: &[u8]) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + let val = tx.get(&tree, key)?; + match val { + None => Ok(None), + Some(v) => Ok(Some(v.to_vec())), + } + } + + fn approximate_len(&self, tree_idx: usize) -> Result { + let tree = self.get_tree(tree_idx)?; + Ok(tree.approximate_len()) + } + fn is_empty(&self, tree_idx: usize) -> Result { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(tx.is_empty(&tree)?) + } + + fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree_idx)?; + let mut tx = self.keyspace.write_tx(); + tx.insert(&tree, key, value); + tx.commit()?; + Ok(()) + } + + fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> { + let tree = self.get_tree(tree_idx)?; + let mut tx = self.keyspace.write_tx(); + tx.remove(&tree, key); + tx.commit()?; + Ok(()) + } + + fn clear(&self, tree_idx: usize) -> Result<()> { + let mut trees = self.trees.write(); + + if tree_idx >= trees.len() { + return Err(Error("invalid tree id".into())); + } + let (name, tree) = trees.remove(tree_idx); + + self.keyspace.delete_partition(tree)?; + let tree = self + .keyspace + .open_partition(&name, PartitionCreateOptions::default())?; + trees.insert(tree_idx, (name, tree)); + + Ok(()) + } + + fn iter(&self, tree_idx: usize) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new(tx.iter(&tree).map(iterator_remap))) + } + + fn iter_rev(&self, tree_idx: usize) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap))) + } + + fn range<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new( + tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high)) + .map(iterator_remap), + )) + } + fn range_rev<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new( + tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high)) + .rev() + .map(iterator_remap), + )) + } + + // ---- + + fn transaction(&self, f: &dyn ITxFn) -> TxResult { + let trees = self.trees.read(); + let mut tx = FjallTx { + trees: &trees[..], + tx: self.keyspace.write_tx(), + }; + + let res = f.try_on(&mut tx); + match res { + TxFnResult::Ok(on_commit) => { + tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; + Ok(on_commit) + } + TxFnResult::Abort => { + tx.tx.rollback(); + Err(TxError::Abort(())) + } + TxFnResult::DbErr => { + tx.tx.rollback(); + Err(TxError::Db(Error( + "(this message will be discarded)".into(), + ))) + } + } + } +} + +// ---- + +struct FjallTx<'a> { + trees: &'a [(String, TransactionalPartitionHandle)], + tx: WriteTransaction<'a>, +} + +impl<'a> FjallTx<'a> { + fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> { + self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| { + TxOpError(Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + )) + }) + } +} + +impl<'a> ITx for FjallTx<'a> { + fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + match self.tx.get(tree, key)? { + Some(v) => Ok(Some(v.to_vec())), + None => Ok(None), + } + } + fn len(&self, tree_idx: usize) -> TxOpResult { + let tree = self.get_tree(tree_idx)?; + Ok(self.tx.len(tree)? as usize) + } + + fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { + let tree = self.get_tree(tree_idx)?.clone(); + self.tx.insert(&tree, key, value); + Ok(()) + } + fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> { + let tree = self.get_tree(tree_idx)?.clone(); + self.tx.remove(&tree, key); + Ok(()) + } + fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> { + unimplemented!("LSM tree clearing in cross-partition transaction is not supported") + } + + fn iter(&self, tree_idx: usize) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?.clone(); + Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx))) + } + fn iter_rev(&self, tree_idx: usize) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?.clone(); + Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx))) + } + + fn range<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + let low = clone_bound(low); + let high = clone_bound(high); + Ok(Box::new( + self.tx + .range::, ByteVecRangeBounds>(&tree, (low, high)) + .map(iterator_remap_tx), + )) + } + fn range_rev<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + let low = clone_bound(low); + let high = clone_bound(high); + Ok(Box::new( + self.tx + .range::, ByteVecRangeBounds>(&tree, (low, high)) + .rev() + .map(iterator_remap_tx), + )) + } +} + +// -- maps fjall's (k, v) to ours + +fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> { + r.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| e.into()) +} + +fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> { + r.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| e.into()) +} + +// -- utils to deal with Garage's tightness on Bound lifetimes + +type ByteVecBound = Bound>; +type ByteVecRangeBounds = (ByteVecBound, ByteVecBound); + +fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound { + let value = match bound { + Bound::Excluded(v) | Bound::Included(v) => v.to_vec(), + Bound::Unbounded => vec![], + }; + + match bound { + Bound::Included(_) => Bound::Included(value), + Bound::Excluded(_) => Bound::Excluded(value), + Bound::Unbounded => Bound::Unbounded, + } +} + +// -- utils to encode table names -- + +fn encode_name(s: &str) -> Result { + let base = 'A' as u32; + + let mut ret = String::with_capacity(s.len() + 10); + for c in s.chars() { + if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' { + ret.push(c); + } else if c <= u8::MAX as char { + ret.push('$'); + let c_hi = c as u32 / 16; + let c_lo = c as u32 % 16; + ret.push(char::from_u32(base + c_hi).unwrap()); + ret.push(char::from_u32(base + c_lo).unwrap()); + } else { + return Err(Error( + format!("table name {} could not be safely encoded", s).into(), + )); + } + } + Ok(ret) +} + +fn decode_name(s: &str) -> Result { + use std::convert::TryFrom; + + let errfn = || Error(format!("encoded table name {} is invalid", s).into()); + let c_map = |c: char| { + let c = c as u32; + let base = 'A' as u32; + if (base..base + 16).contains(&c) { + Some(c - base) + } else { + None + } + }; + + let mut ret = String::with_capacity(s.len()); + let mut it = s.chars(); + while let Some(c) = it.next() { + if c == '$' { + let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?; + let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?; + let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?; + ret.push(c_dec); + } else { + ret.push(c); + } + } + Ok(ret) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encdec_name() { + for name in [ + "testname", + "test_name", + "test name", + "test$name", + "test:name@help.me$get/this**right", + ] { + let encname = encode_name(name).unwrap(); + assert!(!encname.contains(' ')); + assert!(!encname.contains('.')); + assert!(!encname.contains('*')); + assert_eq!(*name, decode_name(&encname).unwrap()); + } + } +} diff --git a/src/db/lib.rs b/src/db/lib.rs index fe44b01e..2a467c7c 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -1,15 +1,14 @@ #[macro_use] -#[cfg(feature = "sqlite")] extern crate tracing; +#[cfg(feature = "fjall")] +pub mod fjall_adapter; #[cfg(feature = "lmdb")] pub mod lmdb_adapter; -#[cfg(feature = "sled")] -pub mod sled_adapter; #[cfg(feature = "sqlite")] pub mod sqlite_adapter; -pub mod counted_tree_hack; +pub mod open; #[cfg(test)] pub mod test; @@ -18,9 +17,12 @@ use core::ops::{Bound, RangeBounds}; use std::borrow::Cow; use std::cell::Cell; +use std::path::PathBuf; use std::sync::Arc; -use err_derive::Error; +use thiserror::Error; + +pub use open::*; pub(crate) type OnCommit = Vec>; @@ -42,16 +44,23 @@ pub type TxValueIter<'a> = Box); +impl From for Error { + fn from(e: std::io::Error) -> Error { + Error(format!("IO: {}", e).into()) + } +} + pub type Result = std::result::Result; #[derive(Debug, Error)] -#[error(display = "{}", _0)] +#[error("{0}")] pub struct TxOpError(pub(crate) Error); pub type TxOpResult = std::result::Result; +#[derive(Debug)] pub enum TxError { Abort(E), Db(Error), @@ -97,35 +106,51 @@ impl Db { result: Cell::new(None), }; let tx_res = self.0.transaction(&f); - let ret = f - .result - .into_inner() - .expect("Transaction did not store result"); + let fn_res = f.result.into_inner(); - match tx_res { - Ok(on_commit) => match ret { - Ok(value) => { - on_commit.into_iter().for_each(|f| f()); - Ok(value) - } - _ => unreachable!(), - }, - Err(TxError::Abort(())) => match ret { - Err(TxError::Abort(e)) => Err(TxError::Abort(e)), - _ => unreachable!(), - }, - Err(TxError::Db(e2)) => match ret { - // Ok was stored -> the error occured when finalizing - // transaction - Ok(_) => Err(TxError::Db(e2)), - // An error was already stored: that's the one we want to - // return - Err(TxError::Db(e)) => Err(TxError::Db(e)), - _ => unreachable!(), - }, + match (tx_res, fn_res) { + (Ok(on_commit), Some(Ok(value))) => { + // Transaction succeeded + // TxFn stored the value to return to the user in fn_res + // tx_res contains the on_commit list of callbacks, run them now + on_commit.into_iter().for_each(|f| f()); + Ok(value) + } + (Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => { + // Transaction was aborted by user code + // The abort error value is stored in fn_res + Err(TxError::Abort(e)) + } + (Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => { + // Transaction encountered a DB error in user code + // The error value encountered is the one in fn_res, + // tx_res contains only a dummy error message + Err(TxError::Db(fn_e)) + } + (Err(TxError::Db(tx_e)), None) => { + // Transaction encounterred a DB error when initializing the transaction, + // before user code was called + Err(TxError::Db(tx_e)) + } + (Err(TxError::Db(tx_e)), Some(Ok(_))) => { + // Transaction encounterred a DB error when commiting the transaction, + // after user code was called + Err(TxError::Db(tx_e)) + } + (tx_res, fn_res) => { + panic!( + "unexpected error case: tx_res={:?}, fn_res={:?}", + tx_res.map(|_| "..."), + fn_res.map(|x| x.map(|_| "...").map_err(|_| "...")) + ); + } } } + pub fn snapshot(&self, path: &PathBuf) -> Result<()> { + self.0.snapshot(path) + } + pub fn import(&self, other: &Db) -> Result<()> { let existing_trees = self.list_trees()?; if !existing_trees.is_empty() { @@ -141,7 +166,7 @@ impl Db { let tree_names = other.list_trees()?; for name in tree_names { let tree = self.open_tree(&name)?; - if tree.len()? > 0 { + if !tree.is_empty()? { return Err(Error(format!("tree {} already contains data", name).into())); } @@ -183,12 +208,12 @@ impl Tree { self.0.get(self.1, key.as_ref()) } #[inline] - pub fn len(&self) -> Result { - self.0.len(self.1) + pub fn approximate_len(&self) -> Result { + self.0.approximate_len(self.1) } #[inline] - pub fn fast_len(&self) -> Result> { - self.0.fast_len(self.1) + pub fn is_empty(&self) -> Result { + self.0.is_empty(self.1) } #[inline] @@ -204,16 +229,12 @@ impl Tree { /// Returns the old value if there was one #[inline] - pub fn insert, U: AsRef<[u8]>>( - &self, - key: T, - value: U, - ) -> Result> { + pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } /// Returns the old value if there was one #[inline] - pub fn remove>(&self, key: T) -> Result> { + pub fn remove>(&self, key: T) -> Result<()> { self.0.remove(self.1, key.as_ref()) } /// Clears all values from the tree @@ -271,14 +292,19 @@ impl<'a> Transaction<'a> { tree: &Tree, key: T, value: U, - ) -> TxOpResult> { + ) -> TxOpResult<()> { self.tx.insert(tree.1, key.as_ref(), value.as_ref()) } /// Returns the old value if there was one #[inline] - pub fn remove>(&mut self, tree: &Tree, key: T) -> TxOpResult> { + pub fn remove>(&mut self, tree: &Tree, key: T) -> TxOpResult<()> { self.tx.remove(tree.1, key.as_ref()) } + /// Clears all values in a tree + #[inline] + pub fn clear(&mut self, tree: &Tree) -> TxOpResult<()> { + self.tx.clear(tree.1) + } #[inline] pub fn iter(&self, tree: &Tree) -> TxOpResult> { @@ -322,15 +348,14 @@ pub(crate) trait IDb: Send + Sync { fn engine(&self) -> String; fn open_tree(&self, name: &str) -> Result; fn list_trees(&self) -> Result>; + fn snapshot(&self, path: &PathBuf) -> Result<()>; fn get(&self, tree: usize, key: &[u8]) -> Result>; - fn len(&self, tree: usize) -> Result; - fn fast_len(&self, _tree: usize) -> Result> { - Ok(None) - } + fn approximate_len(&self, tree: usize) -> Result; + fn is_empty(&self, tree: usize) -> Result; - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result>; - fn remove(&self, tree: usize, key: &[u8]) -> Result>; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn remove(&self, tree: usize, key: &[u8]) -> Result<()>; fn clear(&self, tree: usize) -> Result<()>; fn iter(&self, tree: usize) -> Result>; @@ -356,8 +381,9 @@ pub(crate) trait ITx { fn get(&self, tree: usize, key: &[u8]) -> TxOpResult>; fn len(&self, tree: usize) -> TxOpResult; - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult>; - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult>; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()>; + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()>; + fn clear(&mut self, tree: usize) -> TxOpResult<()>; fn iter(&self, tree: usize) -> TxOpResult>; fn iter_rev(&self, tree: usize) -> TxOpResult>; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 59fa132d..ac185ae9 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -1,20 +1,65 @@ use core::ops::Bound; -use core::ptr::NonNull; use std::collections::HashMap; use std::convert::TryInto; +use std::marker::PhantomPinned; +use std::path::PathBuf; +use std::pin::Pin; use std::sync::{Arc, RwLock}; use heed::types::ByteSlice; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ + open::{Engine, OpenOpt}, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, TxValueIter, Value, ValueIter, }; pub use heed; +// ---- top-level open function + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening LMDB database at: {}", path.display()); + if let Err(e) = std::fs::create_dir_all(&path) { + return Err(Error( + format!("Unable to create LMDB data directory: {}", e).into(), + )); + } + + let map_size = match opt.lmdb_map_size { + None => recommended_map_size(), + Some(v) => v - (v % 4096), + }; + + let mut env_builder = heed::EnvOpenOptions::new(); + env_builder.max_dbs(100); + env_builder.map_size(map_size); + env_builder.max_readers(2048); + unsafe { + env_builder.flag(heed::flags::Flags::MdbNoRdAhead); + env_builder.flag(heed::flags::Flags::MdbNoMetaSync); + if !opt.fsync { + env_builder.flag(heed::flags::Flags::MdbNoSync); + } + } + match env_builder.open(&path) { + Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => { + return Err(Error( + "OutOfMemory error while trying to open LMDB database. This can happen \ + if your operating system is not allowing you to use sufficient virtual \ + memory address space. Please check that no limit is set (ulimit -v). \ + You may also try to set a smaller `lmdb_map_size` configuration parameter. \ + On 32-bit machines, you should probably switch to another database engine." + .into(), + )) + } + Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())), + Ok(db) => Ok(LmdbDb::init(db)), + } +} + // -- err impl From for Error { @@ -102,6 +147,14 @@ impl IDb for LmdbDb { Ok(ret2) } + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Lmdb.db_path(base_path); + self.db + .copy_to_path(path, heed::CompactionOption::Enabled)?; + Ok(()) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result> { @@ -115,32 +168,31 @@ impl IDb for LmdbDb { } } - fn len(&self, tree: usize) -> Result { + fn approximate_len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; Ok(tree.len(&tx)?.try_into().unwrap()) } - - fn fast_len(&self, tree: usize) -> Result> { - Ok(Some(self.len(tree)?)) + fn is_empty(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + Ok(tree.is_empty(&tx)?) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - let old_val = tree.get(&tx, key)?.map(Vec::from); tree.put(&mut tx, key, value)?; tx.commit()?; - Ok(old_val) + Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result> { + fn remove(&self, tree: usize, key: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - let old_val = tree.get(&tx, key)?.map(Vec::from); tree.delete(&mut tx, key)?; tx.commit()?; - Ok(old_val) + Ok(()) } fn clear(&self, tree: usize) -> Result<()> { @@ -154,13 +206,15 @@ impl IDb for LmdbDb { fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) } } fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) } } fn range<'r>( @@ -171,7 +225,8 @@ impl IDb for LmdbDb { ) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) } } fn range_rev<'r>( &self, @@ -181,7 +236,8 @@ impl IDb for LmdbDb { ) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) } } // ---- @@ -228,7 +284,7 @@ impl<'a> LmdbTx<'a> { fn get_tree(&self, i: usize) -> TxOpResult<&Database> { self.trees.get(i).ok_or_else(|| { TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), + "invalid tree id (it might have been opened after the transaction started)".into(), )) }) } @@ -242,49 +298,63 @@ impl<'a> ITx for LmdbTx<'a> { None => Ok(None), } } - fn len(&self, _tree: usize) -> TxOpResult { - unimplemented!(".len() in transaction not supported with LMDB backend") + fn len(&self, tree: usize) -> TxOpResult { + let tree = self.get_tree(tree)?; + Ok(tree.len(&self.tx)? as usize) } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { let tree = *self.get_tree(tree)?; - let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.put(&mut self.tx, key, value)?; - Ok(old_val) + Ok(()) } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> { let tree = *self.get_tree(tree)?; - let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.delete(&mut self.tx, key)?; - Ok(old_val) + Ok(()) + } + fn clear(&mut self, tree: usize) -> TxOpResult<()> { + let tree = *self.get_tree(tree)?; + tree.clear(&mut self.tx)?; + Ok(()) } - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + fn iter(&self, tree: usize) -> TxOpResult> { + let tree = *self.get_tree(tree)?; + Ok(Box::new(tree.iter(&self.tx)?.map(tx_iter_item))) } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + fn iter_rev(&self, tree: usize) -> TxOpResult> { + let tree = *self.get_tree(tree)?; + Ok(Box::new(tree.rev_iter(&self.tx)?.map(tx_iter_item))) } fn range<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + let tree = *self.get_tree(tree)?; + Ok(Box::new( + tree.range(&self.tx, &(low, high))?.map(tx_iter_item), + )) } fn range_rev<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + let tree = *self.get_tree(tree)?; + Ok(Box::new( + tree.rev_range(&self.tx, &(low, high))?.map(tx_iter_item), + )) } } -// ---- +// ---- iterators outside transactions ---- +// complicated, they must hold the transaction object +// therefore a bit of unsafe code (it is a self-referential struct) type IteratorItem<'a> = heed::Result<( >::DItem, @@ -297,22 +367,43 @@ where { tx: RoTxn<'a>, iter: Option, + _pin: PhantomPinned, } impl<'a, I> TxAndIterator<'a, I> where I: Iterator> + 'a, { - fn make(tx: RoTxn<'a>, iterfun: F) -> Result> + fn iter(self: Pin<&mut Self>) -> &mut Option { + // Safety: iter is not structural + unsafe { &mut self.get_unchecked_mut().iter } + } + + /// Safety: iterfun must not store its argument anywhere but in its result. + unsafe fn make(tx: RoTxn<'a>, iterfun: F) -> Result> where F: FnOnce(&'a RoTxn<'a>) -> Result, { - let mut res = TxAndIterator { tx, iter: None }; + let res = TxAndIterator { + tx, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); - let tx = unsafe { NonNull::from(&res.tx).as_ref() }; - res.iter = Some(iterfun(tx)?); + let tx_lifetime_overextended: &'a RoTxn<'a> = { + let tx = &boxed.tx; + // Safety: Artificially extending the lifetime because + // this reference will only be stored and accessed from the + // returned ValueIter which guarantees that it is destroyed + // before the tx it is pointing to. + unsafe { &*&raw const *tx } + }; + let iter = iterfun(&tx_lifetime_overextended)?; - Ok(Box::new(res)) + *boxed.as_mut().iter() = Some(iter); + + Ok(Box::new(TxAndIteratorPin(boxed))) } } @@ -321,26 +412,44 @@ where I: Iterator> + 'a, { fn drop(&mut self) { - drop(self.iter.take()); + // Safety: `new_unchecked` is okay because we know this value is never + // used again after being dropped. + let this = unsafe { Pin::new_unchecked(self) }; + drop(this.iter().take()); } } -impl<'a, I> Iterator for TxAndIterator<'a, I> +struct TxAndIteratorPin<'a, I>(Pin>>) +where + I: Iterator> + 'a; + +impl<'a, I> Iterator for TxAndIteratorPin<'a, I> where I: Iterator> + 'a, { type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { - match self.iter.as_mut().unwrap().next() { - None => None, - Some(Err(e)) => Some(Err(e.into())), - Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), - } + let mut_ref = Pin::as_mut(&mut self.0); + let next = mut_ref.iter().as_mut()?.next()?; + let res = match next { + Err(e) => Err(e.into()), + Ok((k, v)) => Ok((k.to_vec(), v.to_vec())), + }; + Some(res) } } -// ---- +// ---- iterators within transactions ---- + +fn tx_iter_item<'a>( + item: std::result::Result<(&'a [u8], &'a [u8]), heed::Error>, +) -> TxOpResult<(Vec, Vec)> { + item.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| TxOpError(Error::from(e))) +} + +// ---- utility ---- #[cfg(target_pointer_width = "64")] pub fn recommended_map_size() -> usize { diff --git a/src/db/open.rs b/src/db/open.rs new file mode 100644 index 00000000..23391c61 --- /dev/null +++ b/src/db/open.rs @@ -0,0 +1,109 @@ +use std::path::PathBuf; + +use crate::{Db, Error, Result}; + +/// List of supported database engine types +/// +/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter +/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine +/// and valid engine, whose support is not enabled via feature flag. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Engine { + Lmdb, + Sqlite, + Fjall, +} + +impl Engine { + /// Return variant name as static `&str` + pub fn as_str(&self) -> &'static str { + match self { + Self::Lmdb => "lmdb", + Self::Sqlite => "sqlite", + Self::Fjall => "fjall", + } + } + + /// Return engine-specific DB path from base path + pub fn db_path(&self, base_path: &PathBuf) -> PathBuf { + let mut ret = base_path.clone(); + match self { + Self::Lmdb => { + ret.push("db.lmdb"); + } + Self::Sqlite => { + ret.push("db.sqlite"); + } + Self::Fjall => { + ret.push("db.fjall"); + } + } + ret + } +} + +impl std::fmt::Display for Engine { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.as_str().fmt(fmt) + } +} + +impl std::str::FromStr for Engine { + type Err = Error; + + fn from_str(text: &str) -> Result { + match text { + "lmdb" | "heed" => Ok(Self::Lmdb), + "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite), + "fjall" => Ok(Self::Fjall), + "sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())), + kind => Err(Error( + format!( + "Invalid DB engine: {} (options are: lmdb, sqlite, fjall)", + kind + ) + .into(), + )), + } + } +} + +pub struct OpenOpt { + pub fsync: bool, + pub lmdb_map_size: Option, + pub fjall_block_cache_size: Option, +} + +impl Default for OpenOpt { + fn default() -> Self { + Self { + fsync: false, + lmdb_map_size: None, + fjall_block_cache_size: None, + } + } +} + +pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result { + match engine { + // ---- Sqlite DB ---- + #[cfg(feature = "sqlite")] + Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt), + + // ---- LMDB DB ---- + #[cfg(feature = "lmdb")] + Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt), + + // ---- Fjall DB ---- + #[cfg(feature = "fjall")] + Engine::Fjall => crate::fjall_adapter::open_db(path, opt), + + // Pattern is unreachable when all supported DB engines are compiled into binary. The allow + // attribute is added so that we won't have to change this match in case stop building + // support for one or more engines by default. + #[allow(unreachable_patterns)] + engine => Err(Error( + format!("DB engine support not available in this build: {}", engine).into(), + )), + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs deleted file mode 100644 index 84f2001b..00000000 --- a/src/db/sled_adapter.rs +++ /dev/null @@ -1,274 +0,0 @@ -use core::ops::Bound; - -use std::cell::Cell; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use sled::transaction::{ - ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, - UnabortableTransactionError, -}; - -use crate::{ - Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, - TxResult, TxValueIter, Value, ValueIter, -}; - -pub use sled; - -// -- err - -impl From for Error { - fn from(e: sled::Error) -> Error { - Error(format!("Sled: {}", e).into()) - } -} - -impl From for TxOpError { - fn from(e: sled::Error) -> TxOpError { - TxOpError(e.into()) - } -} - -// -- db - -pub struct SledDb { - db: sled::Db, - trees: RwLock<(Vec, HashMap)>, -} - -impl SledDb { - #[deprecated( - since = "0.9.0", - note = "The Sled database is now deprecated and will be removed in Garage v1.0. Please migrate to LMDB or Sqlite as soon as possible." - )] - pub fn init(db: sled::Db) -> Db { - tracing::warn!("-------------------- IMPORTANT WARNING !!! ----------------------"); - tracing::warn!("The Sled database is now deprecated and will be removed in Garage v1.0."); - tracing::warn!("Please migrate to LMDB or Sqlite as soon as possible."); - tracing::warn!("-----------------------------------------------------------------------"); - let s = Self { - db, - trees: RwLock::new((Vec::new(), HashMap::new())), - }; - Db(Arc::new(s)) - } - - fn get_tree(&self, i: usize) -> Result { - self.trees - .read() - .unwrap() - .0 - .get(i) - .cloned() - .ok_or_else(|| Error("invalid tree id".into())) - } -} - -impl IDb for SledDb { - fn engine(&self) -> String { - "Sled".into() - } - - fn open_tree(&self, name: &str) -> Result { - let mut trees = self.trees.write().unwrap(); - if let Some(i) = trees.1.get(name) { - Ok(*i) - } else { - let tree = self.db.open_tree(name)?; - let i = trees.0.len(); - trees.0.push(tree); - trees.1.insert(name.to_string(), i); - Ok(i) - } - } - - fn list_trees(&self) -> Result> { - let mut trees = vec![]; - for name in self.db.tree_names() { - let name = std::str::from_utf8(&name) - .map_err(|e| Error(format!("{}", e).into()))? - .to_string(); - if name != "__sled__default" { - trees.push(name); - } - } - Ok(trees) - } - - // ---- - - fn get(&self, tree: usize, key: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let val = tree.get(key)?; - Ok(val.map(|x| x.to_vec())) - } - - fn len(&self, tree: usize) -> Result { - let tree = self.get_tree(tree)?; - Ok(tree.len()) - } - - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let old_val = tree.insert(key, value)?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn remove(&self, tree: usize, key: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let old_val = tree.remove(key)?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn clear(&self, tree: usize) -> Result<()> { - let tree = self.get_tree(tree)?; - tree.clear()?; - Ok(()) - } - - fn iter(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - - fn iter_rev(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - - fn range<'r>( - &self, - tree: usize, - low: Bound<&'r [u8]>, - high: Bound<&'r [u8]>, - ) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - fn range_rev<'r>( - &self, - tree: usize, - low: Bound<&'r [u8]>, - high: Bound<&'r [u8]>, - ) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( - |v| v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into), - ))) - } - - // ---- - - fn transaction(&self, f: &dyn ITxFn) -> TxResult { - let trees = self.trees.read().unwrap(); - let res = trees.0.transaction(|txtrees| { - let mut tx = SledTx { - trees: txtrees, - err: Cell::new(None), - }; - match f.try_on(&mut tx) { - TxFnResult::Ok(on_commit) => { - assert!(tx.err.into_inner().is_none()); - Ok(on_commit) - } - TxFnResult::Abort => { - assert!(tx.err.into_inner().is_none()); - Err(ConflictableTransactionError::Abort(())) - } - TxFnResult::DbErr => { - let e = tx.err.into_inner().expect("No DB error"); - Err(e.into()) - } - } - }); - match res { - Ok(on_commit) => Ok(on_commit), - Err(TransactionError::Abort(())) => Err(TxError::Abort(())), - Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), - } - } -} - -// ---- - -struct SledTx<'a> { - trees: &'a [TransactionalTree], - err: Cell>, -} - -impl<'a> SledTx<'a> { - fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalTree> { - self.trees.get(i).ok_or_else(|| { - TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), - )) - }) - } - - fn save_error( - &self, - v: std::result::Result, - ) -> TxOpResult { - match v { - Ok(x) => Ok(x), - Err(e) => { - let txt = format!("{}", e); - self.err.set(Some(e)); - Err(TxOpError(Error(txt.into()))) - } - } - } -} - -impl<'a> ITx for SledTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let tmp = self.save_error(tree.get(key))?; - Ok(tmp.map(|x| x.to_vec())) - } - fn len(&self, _tree: usize) -> TxOpResult { - unimplemented!(".len() in transaction not supported with Sled backend") - } - - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let old_val = self.save_error(tree.insert(key, value))?; - Ok(old_val.map(|x| x.to_vec())) - } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let old_val = self.save_error(tree.remove(key))?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - - fn range<'r>( - &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, - ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - fn range_rev<'r>( - &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, - ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } -} diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 9f967c66..a03ee8ef 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -1,20 +1,35 @@ use core::ops::Bound; -use std::borrow::BorrowMut; use std::marker::PhantomPinned; +use std::path::PathBuf; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, RwLock}; -use rusqlite::{params, Connection, Rows, Statement, Transaction}; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::{params, Rows, Statement, Transaction}; use crate::{ + open::{Engine, OpenOpt}, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, TxValueIter, Value, ValueIter, }; pub use rusqlite; +// ---- top-level open function + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening Sqlite database at: {}", path.display()); + let manager = r2d2_sqlite::SqliteConnectionManager::file(path); + Ok(SqliteDb::new(manager, opt.fsync)?) +} + +// ---- + +type Connection = r2d2::PooledConnection; + // --- err impl From for Error { @@ -23,6 +38,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: r2d2::Error) -> Error { + Error(format!("Sqlite: {}", e).into()) + } +} + impl From for TxOpError { fn from(e: rusqlite::Error) -> TxOpError { TxOpError(e.into()) @@ -31,35 +52,47 @@ impl From for TxOpError { // -- db -pub struct SqliteDb(Mutex); - -struct SqliteDbInner { - db: Connection, - trees: Vec, +pub struct SqliteDb { + db: Pool, + trees: RwLock>>, + // All operations that might write on the DB must take this lock first. + // This emulates LMDB's approach where a single writer can be + // active at once. + write_lock: Mutex<()>, } impl SqliteDb { - pub fn init(db: rusqlite::Connection) -> Db { - let s = Self(Mutex::new(SqliteDbInner { - db, - trees: Vec::new(), - })); - Db(Arc::new(s)) + pub fn new(manager: SqliteConnectionManager, sync_mode: bool) -> Result { + let manager = manager.with_init(move |db| { + db.pragma_update(None, "journal_mode", "WAL")?; + if sync_mode { + db.pragma_update(None, "synchronous", "NORMAL")?; + } else { + db.pragma_update(None, "synchronous", "OFF")?; + } + Ok(()) + }); + let s = Self { + db: Pool::builder().build(manager)?, + trees: RwLock::new(vec![]), + write_lock: Mutex::new(()), + }; + Ok(Db(Arc::new(s))) } } -impl SqliteDbInner { - fn get_tree(&self, i: usize) -> Result<&'_ str> { +impl SqliteDb { + fn get_tree(&self, i: usize) -> Result> { self.trees + .read() + .unwrap() .get(i) - .map(String::as_str) + .cloned() .ok_or_else(|| Error("invalid tree id".into())) } - fn internal_get(&self, tree: &str, key: &[u8]) -> Result> { - let mut stmt = self - .db - .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> Result> { + let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; let mut res_iter = stmt.query([key])?; match res_iter.next()? { None => Ok(None), @@ -75,13 +108,14 @@ impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { let name = format!("tree_{}", name.replace(':', "_COLON_")); - let mut this = self.0.lock().unwrap(); + let mut trees = self.trees.write().unwrap(); - if let Some(i) = this.trees.iter().position(|x| x == &name) { + if let Some(i) = trees.iter().position(|x| x.as_ref() == &name) { Ok(i) } else { + let db = self.db.get()?; trace!("create table {}", name); - this.db.execute( + db.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BLOB PRIMARY KEY, @@ -93,8 +127,8 @@ impl IDb for SqliteDb { )?; trace!("table created: {}, unlocking", name); - let i = this.trees.len(); - this.trees.push(name.to_string()); + let i = trees.len(); + trees.push(name.to_string().into_boxed_str().into()); Ok(i) } } @@ -102,11 +136,8 @@ impl IDb for SqliteDb { fn list_trees(&self) -> Result> { let mut trees = vec![]; - trace!("list_trees: lock db"); - let this = self.0.lock().unwrap(); - trace!("list_trees: lock acquired"); - - let mut stmt = this.db.prepare( + let db = self.db.get()?; + let mut stmt = db.prepare( "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", )?; let mut rows = stmt.query([])?; @@ -119,24 +150,33 @@ impl IDb for SqliteDb { Ok(trees) } + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Sqlite + .db_path(&base_path) + .into_os_string() + .into_string() + .map_err(|_| Error("invalid sqlite path string".into()))?; + + info!("Start sqlite VACUUM INTO `{}`", path); + self.db.get()?.execute("VACUUM INTO ?1", params![path])?; + info!("Finished sqlite VACUUM INTO `{}`", path); + + Ok(()) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result> { - trace!("get {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("get {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; - this.internal_get(tree, key) + let tree = self.get_tree(tree)?; + self.internal_get(&self.db.get()?, &tree, key) } - fn len(&self, tree: usize) -> Result { - trace!("len {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("len {}: lock acquired", tree); + fn approximate_len(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; - let tree = this.get_tree(tree)?; - let mut stmt = this.db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; + let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; match res_iter.next()? { None => Ok(0), @@ -144,74 +184,60 @@ impl IDb for SqliteDb { } } - fn fast_len(&self, tree: usize) -> Result> { - Ok(Some(self.len(tree)?)) + fn is_empty(&self, tree: usize) -> Result { + Ok(self.approximate_len(tree)? == 0) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { - trace!("insert {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("insert {}: lock acquired", tree); + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - let old_val = this.internal_get(tree, key)?; + let old_val = self.internal_get(&db, &tree, key)?; let sql = match &old_val { Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), }; - let n = this.db.execute(&sql, params![key, value])?; + let n = db.execute(&sql, params![key, value])?; assert_eq!(n, 1); - Ok(old_val) + drop(lock); + Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result> { - trace!("remove {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("remove {}: lock acquired", tree); + fn remove(&self, tree: usize, key: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - let old_val = this.internal_get(tree, key)?; + db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - if old_val.is_some() { - let n = this - .db - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - assert_eq!(n, 1); - } - - Ok(old_val) + drop(lock); + Ok(()) } fn clear(&self, tree: usize) -> Result<()> { - trace!("clear {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("clear {}: lock acquired", tree); + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - this.db.execute(&format!("DELETE FROM {}", tree), [])?; + db.execute(&format!("DELETE FROM {}", tree), [])?; + + drop(lock); Ok(()) } fn iter(&self, tree: usize) -> Result> { - trace!("iter {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("iter {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - DbValueIterator::make(this, &sql, []) + DbValueIterator::make(self.db.get()?, &sql, []) } fn iter_rev(&self, tree: usize) -> Result> { - trace!("iter_rev {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("iter_rev {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); - DbValueIterator::make(this, &sql, []) + DbValueIterator::make(self.db.get()?, &sql, []) } fn range<'r>( @@ -220,11 +246,7 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - trace!("range {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("range {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); @@ -234,7 +256,7 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref()) } fn range_rev<'r>( &self, @@ -242,11 +264,7 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - trace!("range_rev {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("range_rev {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); @@ -256,25 +274,20 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref()) } // ---- fn transaction(&self, f: &dyn ITxFn) -> TxResult { - trace!("transaction: lock db"); - let mut this = self.0.lock().unwrap(); - trace!("transaction: lock acquired"); - - let this_mut_ref: &mut SqliteDbInner = this.borrow_mut(); + let mut db = self.db.get().map_err(Error::from).map_err(TxError::Db)?; + let trees = self.trees.read().unwrap(); + let lock = self.write_lock.lock(); + trace!("trying transaction"); let mut tx = SqliteTx { - tx: this_mut_ref - .db - .transaction() - .map_err(Error::from) - .map_err(TxError::Db)?, - trees: &this_mut_ref.trees, + tx: db.transaction().map_err(Error::from).map_err(TxError::Db)?, + trees: &trees, }; let res = match f.try_on(&mut tx) { TxFnResult::Ok(on_commit) => { @@ -294,7 +307,8 @@ impl IDb for SqliteDb { }; trace!("transaction done"); - res + drop(lock); + return res; } } @@ -302,14 +316,14 @@ impl IDb for SqliteDb { struct SqliteTx<'a> { tx: Transaction<'a>, - trees: &'a [String], + trees: &'a [Arc], } impl<'a> SqliteTx<'a> { fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> { - self.trees.get(i).map(String::as_ref).ok_or_else(|| { + self.trees.get(i).map(Arc::as_ref).ok_or_else(|| { TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), + "invalid tree id (it might have been opened after the transaction started)".into(), )) }) } @@ -341,73 +355,86 @@ impl<'a> ITx for SqliteTx<'a> { } } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { let tree = self.get_tree(tree)?; - let old_val = self.internal_get(tree, key)?; - - let sql = match &old_val { - Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), - None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), - }; - let n = self.tx.execute(&sql, params![key, value])?; - assert_eq!(n, 1); - - Ok(old_val) + let sql = format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree); + self.tx.execute(&sql, params![key, value])?; + Ok(()) } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> { let tree = self.get_tree(tree)?; - let old_val = self.internal_get(tree, key)?; - - if old_val.is_some() { - let n = self - .tx - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - assert_eq!(n, 1); - } - - Ok(old_val) + self.tx + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + Ok(()) + } + fn clear(&mut self, tree: usize) -> TxOpResult<()> { + let tree = self.get_tree(tree)?; + self.tx.execute(&format!("DELETE FROM {}", tree), [])?; + Ok(()) } - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!(); + fn iter(&self, tree: usize) -> TxOpResult> { + let tree = self.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); + TxValueIterator::make(self, &sql, []) } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!(); + fn iter_rev(&self, tree: usize) -> TxOpResult> { + let tree = self.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); + TxValueIterator::make(self, &sql, []) } fn range<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!(); + let tree = self.get_tree(tree)?; + + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + + TxValueIterator::make::<&[&dyn rusqlite::ToSql]>(self, &sql, params.as_ref()) } fn range_rev<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!(); + let tree = self.get_tree(tree)?; + + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + + TxValueIterator::make::<&[&dyn rusqlite::ToSql]>(self, &sql, params.as_ref()) } } -// ---- +// ---- iterators outside transactions ---- +// complicated, they must hold the Statement and Row objects +// therefore quite some unsafe code (it is a self-referential struct) struct DbValueIterator<'a> { - db: MutexGuard<'a, SqliteDbInner>, + db: Connection, stmt: Option>, iter: Option>, _pin: PhantomPinned, } impl<'a> DbValueIterator<'a> { - fn make( - db: MutexGuard<'a, SqliteDbInner>, - sql: &str, - args: P, - ) -> Result> { + fn make(db: Connection, sql: &str, args: P) -> Result> { let res = DbValueIterator { db, stmt: None, @@ -417,17 +444,23 @@ impl<'a> DbValueIterator<'a> { let mut boxed = Box::pin(res); trace!("make iterator with sql: {}", sql); + // This unsafe allows us to bypass lifetime checks + let db = unsafe { NonNull::from(&boxed.db).as_ref() }; + let stmt = db.prepare(sql)?; + + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct unsafe { - let db = NonNull::from(&boxed.db); - let stmt = db.as_ref().db.prepare(sql)?; - - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); + } - let mut stmt = NonNull::from(&boxed.stmt); - let iter = stmt.as_mut().as_mut().unwrap().query(args)?; + // This unsafe allows us to bypass lifetime checks + let stmt = unsafe { NonNull::from(&boxed.stmt).as_mut() }; + let iter = stmt.as_mut().unwrap().query(args)?; - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct + unsafe { Pin::get_unchecked_mut(mut_ref).iter = Some(iter); } @@ -449,28 +482,73 @@ impl<'a> Iterator for DbValueIteratorPin<'a> { type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { - let next = unsafe { - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut self.0); - Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() - }; - let row = match next { - Err(e) => return Some(Err(e.into())), - Ok(None) => return None, - Ok(Some(r)) => r, - }; - let k = match row.get::<_, Vec>(0) { - Err(e) => return Some(Err(e.into())), - Ok(x) => x, - }; - let v = match row.get::<_, Vec>(1) { - Err(e) => return Some(Err(e.into())), - Ok(y) => y, - }; - Some(Ok((k, v))) + let mut_ref = Pin::as_mut(&mut self.0); + // This unsafe allows us to mutably access the iterator field + let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; + iter_next_row(next) } } -// ---- +// ---- iterators within transactions ---- +// it's the same except we don't hold a mutex guard, +// only a Statement and a Rows object + +struct TxValueIterator<'a> { + stmt: Statement<'a>, + iter: Option>, + _pin: PhantomPinned, +} + +impl<'a> TxValueIterator<'a> { + fn make( + tx: &'a SqliteTx<'a>, + sql: &str, + args: P, + ) -> TxOpResult> { + let stmt = tx.tx.prepare(sql)?; + let res = TxValueIterator { + stmt, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); + trace!("make iterator with sql: {}", sql); + + // This unsafe allows us to bypass lifetime checks + let stmt = unsafe { NonNull::from(&boxed.stmt).as_mut() }; + let iter = stmt.query(args)?; + + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct + unsafe { + Pin::get_unchecked_mut(mut_ref).iter = Some(iter); + } + + Ok(Box::new(TxValueIteratorPin(boxed))) + } +} + +impl<'a> Drop for TxValueIterator<'a> { + fn drop(&mut self) { + trace!("drop iter"); + drop(self.iter.take()); + } +} + +struct TxValueIteratorPin<'a>(Pin>>); + +impl<'a> Iterator for TxValueIteratorPin<'a> { + type Item = TxOpResult<(Value, Value)>; + + fn next(&mut self) -> Option { + let mut_ref = Pin::as_mut(&mut self.0); + // This unsafe allows us to mutably access the iterator field + let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; + iter_next_row(next) + } +} + +// ---- utility ---- fn bounds_sql<'r>(low: Bound<&'r [u8]>, high: Bound<&'r [u8]>) -> (String, Vec>) { let mut sql = String::new(); @@ -510,3 +588,25 @@ fn bounds_sql<'r>(low: Bound<&'r [u8]>, high: Bound<&'r [u8]>) -> (String, Vec( + next_row: rusqlite::Result>, +) -> Option> +where + E: From, +{ + let row = match next_row { + Err(e) => return Some(Err(e.into())), + Ok(None) => return None, + Ok(Some(r)) => r, + }; + let k = match row.get::<_, Vec>(0) { + Err(e) => return Some(Err(e.into())), + Ok(x) => x, + }; + let v = match row.get::<_, Vec>(1) { + Err(e) => return Some(Err(e.into())), + Ok(y) => y, + }; + Some(Ok((k, v))) +} diff --git a/src/db/test.rs b/src/db/test.rs index cd99eafa..977dc965 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -1,7 +1,7 @@ use crate::*; fn test_suite(db: Db) { - let tree = db.open_tree("tree").unwrap(); + let tree = db.open_tree("tree:this_is_a_tree").unwrap(); let ka: &[u8] = &b"test"[..]; let kb: &[u8] = &b"zwello"[..]; @@ -10,13 +10,18 @@ fn test_suite(db: Db) { let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - assert!(tree.insert(ka, va).unwrap().is_none()); + // ---- test simple insert/delete ---- + + assert!(tree.insert(ka, va).is_ok()); assert_eq!(tree.get(ka).unwrap().unwrap(), va); + assert_eq!(tree.iter().unwrap().count(), 1); + + // ---- test transaction logic ---- let res = db.transaction::<_, (), _>(|tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); - assert_eq!(tx.insert(&tree, ka, vb).unwrap().unwrap(), va); + assert_eq!(tx.insert(&tree, ka, vb).unwrap(), ()); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); @@ -28,7 +33,7 @@ fn test_suite(db: Db) { let res = db.transaction::<(), _, _>(|tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); - assert_eq!(tx.insert(&tree, ka, vc).unwrap().unwrap(), vb); + assert_eq!(tx.insert(&tree, ka, vc).unwrap(), ()); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc); @@ -37,13 +42,15 @@ fn test_suite(db: Db) { assert!(matches!(res, Err(TxError::Abort(42)))); assert_eq!(tree.get(ka).unwrap().unwrap(), vb); + // ---- test iteration outside of transactions ---- + let mut iter = tree.iter().unwrap(); let next = iter.next().unwrap().unwrap(); assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); - assert!(tree.insert(kb, vc).unwrap().is_none()); + assert!(tree.insert(kb, vc).is_ok()); assert_eq!(tree.get(kb).unwrap().unwrap(), vc); let mut iter = tree.iter().unwrap(); @@ -73,6 +80,48 @@ fn test_suite(db: Db) { assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); + + // ---- test iteration within transactions ---- + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.iter(&tree).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.range(&tree, kint..).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.range_rev(&tree, ..kint).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.iter_rev(&tree).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); } #[test] @@ -90,22 +139,24 @@ fn test_lmdb_db() { drop(path); } -#[test] -#[cfg(feature = "sled")] -fn test_sled_db() { - use crate::sled_adapter::SledDb; - - let path = mktemp::Temp::new_dir().unwrap(); - let db = SledDb::init(sled::open(path.to_path_buf()).unwrap()); - test_suite(db); - drop(path); -} - #[test] #[cfg(feature = "sqlite")] fn test_sqlite_db() { use crate::sqlite_adapter::SqliteDb; - let db = SqliteDb::init(rusqlite::Connection::open_in_memory().unwrap()); + let manager = r2d2_sqlite::SqliteConnectionManager::memory(); + let db = SqliteDb::new(manager, false).unwrap(); + test_suite(db); +} + +#[test] +#[cfg(feature = "fjall")] +fn test_fjall_db() { + use crate::fjall_adapter::{fjall, FjallDb}; + + let path = mktemp::Temp::new_dir().unwrap(); + let config = fjall::Config::new(path).temporary(true); + let keyspace = config.open_transactional().unwrap(); + let db = FjallDb::init(keyspace); test_suite(db); } diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 88729aaf..a4f695a4 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -23,78 +23,88 @@ path = "tests/lib.rs" [dependencies] format_table.workspace = true garage_db.workspace = true -garage_api.workspace = true +garage_api_admin.workspace = true +garage_api_s3.workspace = true +garage_api_k2v = { workspace = true, optional = true } garage_block.workspace = true garage_model.workspace = true +garage_net.workspace = true garage_rpc.workspace = true garage_table.workspace = true garage_util.workspace = true garage_web.workspace = true -backtrace = "0.3" -bytes = "1.0" -bytesize = "1.2" -timeago = { version = "0.4", default-features = false } -parse_duration = "2.1" -hex = "0.4" -tracing = { version = "0.1" } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -rand = "0.8" -async-trait = "0.1.7" -sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -git-version = "0.3.4" +backtrace.workspace = true +bytes.workspace = true +bytesize.workspace = true +timeago.workspace = true +parse_duration.workspace = true +hex.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +async-trait.workspace = true +sha1.workspace = true +sodiumoxide.workspace = true +structopt.workspace = true +git-version.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -structopt = { version = "0.3", default-features = false } -toml = "0.6" +serde.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +tokio.workspace = true -netapp = "0.5" - -opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } -opentelemetry-prometheus = { version = "0.10", optional = true } -opentelemetry-otlp = { version = "0.10", optional = true } -prometheus = { version = "0.13", optional = true } +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true } +syslog-tracing = { workspace = true, optional = true } +tracing-journald = { workspace = true, optional = true } [dev-dependencies] -aws-config = "0.55.2" -aws-sdk-s3 = "0.28" -chrono = "0.4" -http = "0.2" -hmac = "0.12" -hyper = { version = "0.14", features = ["client", "http1", "runtime"] } -sha2 = "0.10" +garage_api_common.workspace = true -static_init = "1.0" -assert-json-diff = "2.0" -serde_json = "1.0" -base64 = "0.21" +aws-sdk-s3.workspace = true +aws-smithy-runtime.workspace = true +chrono.workspace = true +http.workspace = true +hmac.workspace = true +http-body-util.workspace = true +hyper.workspace = true +hyper-util.workspace = true +mktemp.workspace = true +sha2.workspace = true + + +static_init.workspace = true +assert-json-diff.workspace = true +serde_json.workspace = true +base64.workspace = true +crc32fast.workspace = true k2v-client.workspace = true [features] -default = [ "bundled-libs", "metrics", "sled", "lmdb", "sqlite", "k2v" ] +default = [ "bundled-libs", "metrics", "lmdb", "sqlite", "k2v" ] -k2v = [ "garage_util/k2v", "garage_api/k2v" ] +k2v = [ "garage_util/k2v", "garage_api_k2v" ] -# Database engines, Sled is still our default even though we don't like it -sled = [ "garage_model/sled" ] +# Database engines lmdb = [ "garage_model/lmdb" ] sqlite = [ "garage_model/sqlite" ] +fjall = [ "garage_model/fjall" ] # Automatic registration and discovery via Consul API consul-discovery = [ "garage_rpc/consul-discovery" ] # Automatic registration and discovery via Kubernetes API kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] # Prometheus exporter (/metrics endpoint). -metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ] +metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ] # Exporter for the OpenTelemetry Collector. telemetry-otlp = [ "opentelemetry-otlp" ] +# Logging to syslog +syslog = [ "syslog-tracing" ] +# Logging to journald +journald = [ "tracing-journald" ] # NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # exactly one of them should be enabled. diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs index c4a45738..5f908ce4 100644 --- a/src/garage/admin/block.rs +++ b/src/garage/admin/block.rs @@ -25,8 +25,7 @@ impl AdminRpcHandler { } async fn handle_block_info(&self, hash: &String) -> Result { - let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; - let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; + let hash = self.find_block_hash_by_prefix(hash)?; let refcount = self.garage.block_manager.get_block_rc(&hash)?; let block_refs = self .garage @@ -102,6 +101,7 @@ impl AdminRpcHandler { let mut obj_dels = 0; let mut mpu_dels = 0; let mut ver_dels = 0; + let mut br_dels = 0; for hash in blocks { let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; @@ -132,12 +132,19 @@ impl AdminRpcHandler { ver_dels += 1; } } + if !br.deleted.get() { + let mut br = br; + br.deleted.set(); + self.garage.block_ref_table.insert(&br).await?; + br_dels += 1; + } } } Ok(AdminRpc::Ok(format!( - "Purged {} blocks, {} versions, {} objects, {} multipart uploads", + "Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted", blocks.len(), + br_dels, ver_dels, obj_dels, mpu_dels, @@ -189,4 +196,48 @@ impl AdminRpcHandler { Ok(()) } + + // ---- helper function ---- + fn find_block_hash_by_prefix(&self, prefix: &str) -> Result { + if prefix.len() < 4 { + return Err(Error::BadRequest( + "Please specify at least 4 characters of the block hash".into(), + )); + } + + let prefix_bin = + hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?; + + let iter = self + .garage + .block_ref_table + .data + .store + .range(&prefix_bin[..]..) + .map_err(GarageError::from)?; + let mut found = None; + for item in iter { + let (k, _v) = item.map_err(GarageError::from)?; + let hash = Hash::try_from(&k[..32]).unwrap(); + if &hash.as_slice()[..prefix_bin.len()] != prefix_bin { + break; + } + if hex::encode(hash.as_slice()).starts_with(prefix) { + match &found { + Some(x) if *x == hash => (), + Some(_) => { + return Err(Error::BadRequest(format!( + "Several blocks match prefix `{}`", + prefix + ))); + } + None => { + found = Some(hash); + } + } + } + } + + found.ok_or_else(|| Error::BadRequest("No matching block found".into())) + } } diff --git a/src/garage/admin/bucket.rs b/src/garage/admin/bucket.rs index 0781cb8b..073329c1 100644 --- a/src/garage/admin/bucket.rs +++ b/src/garage/admin/bucket.rs @@ -54,9 +54,8 @@ impl AdminRpcHandler { let bucket_id = self .garage .bucket_helper() - .resolve_global_bucket_name(&query.name) - .await? - .ok_or_bad_request("Bucket not found")?; + .admin_get_existing_matching_bucket(&query.name) + .await?; let bucket = self .garage @@ -70,7 +69,7 @@ impl AdminRpcHandler { .table .get(&bucket_id, &EmptyKey) .await? - .map(|x| x.filtered_values(&self.garage.system.ring.borrow())) + .map(|x| x.filtered_values(&self.garage.system.cluster_layout())) .unwrap_or_default(); let mpu_counters = self @@ -79,7 +78,7 @@ impl AdminRpcHandler { .table .get(&bucket_id, &EmptyKey) .await? - .map(|x| x.filtered_values(&self.garage.system.ring.borrow())) + .map(|x| x.filtered_values(&self.garage.system.cluster_layout())) .unwrap_or_default(); let mut relevant_keys = HashMap::new(); @@ -127,13 +126,15 @@ impl AdminRpcHandler { #[allow(clippy::ptr_arg)] async fn handle_create_bucket(&self, name: &String) -> Result { - if !is_valid_bucket_name(name) { + if !is_valid_bucket_name(name, self.garage.config.allow_punycode) { return Err(Error::BadRequest(format!( "{}: {}", name, INVALID_BUCKET_NAME_MESSAGE ))); } + let helper = self.garage.locked_helper().await; + if let Some(alias) = self.garage.bucket_alias_table.get(&EmptyKey, name).await? { if alias.state.get().is_some() { return Err(Error::BadRequest(format!("Bucket {} already exists", name))); @@ -145,21 +146,18 @@ impl AdminRpcHandler { let bucket = Bucket::new(); self.garage.bucket_table.insert(&bucket).await?; - self.garage - .bucket_helper() - .set_global_bucket_alias(bucket.id, name) - .await?; + helper.set_global_bucket_alias(bucket.id, name).await?; Ok(AdminRpc::Ok(format!("Bucket {} was created.", name))) } async fn handle_delete_bucket(&self, query: &DeleteBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); + let helper = self.garage.locked_helper().await; let bucket_id = helper - .resolve_global_bucket_name(&query.name) - .await? - .ok_or_bad_request("Bucket not found")?; + .bucket() + .admin_get_existing_matching_bucket(&query.name) + .await?; // Get the alias, but keep in minde here the bucket name // given in parameter can also be directly the bucket's ID. @@ -174,7 +172,7 @@ impl AdminRpcHandler { .await?; // Check bucket doesn't have other aliases - let mut bucket = helper.get_existing_bucket(bucket_id).await?; + let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?; let bucket_state = bucket.state.as_option().unwrap(); if bucket_state .aliases @@ -195,7 +193,7 @@ impl AdminRpcHandler { } // Check bucket is empty - if !helper.is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(bucket_id).await? { return Err(Error::BadRequest(format!( "Bucket {} is not empty", query.name @@ -231,16 +229,15 @@ impl AdminRpcHandler { } async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); + let helper = self.garage.locked_helper().await; let bucket_id = helper - .resolve_global_bucket_name(&query.existing_bucket) - .await? - .ok_or_bad_request("Bucket not found")?; + .bucket() + .admin_get_existing_matching_bucket(&query.existing_bucket) + .await?; if let Some(key_pattern) = &query.local { - let key = key_helper.get_existing_matching_key(key_pattern).await?; + let key = helper.key().get_existing_matching_key(key_pattern).await?; helper .set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name) @@ -261,11 +258,10 @@ impl AdminRpcHandler { } async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); + let helper = self.garage.locked_helper().await; if let Some(key_pattern) = &query.local { - let key = key_helper.get_existing_matching_key(key_pattern).await?; + let key = helper.key().get_existing_matching_key(key_pattern).await?; let bucket_id = key .state @@ -287,6 +283,7 @@ impl AdminRpcHandler { ))) } else { let bucket_id = helper + .bucket() .resolve_global_bucket_name(&query.name) .await? .ok_or_bad_request("Bucket not found")?; @@ -303,14 +300,14 @@ impl AdminRpcHandler { } async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); + let helper = self.garage.locked_helper().await; let bucket_id = helper - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - let key = key_helper + .bucket() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + let key = helper + .key() .get_existing_matching_key(&query.key_pattern) .await?; @@ -338,14 +335,14 @@ impl AdminRpcHandler { } async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); + let helper = self.garage.locked_helper().await; let bucket_id = helper - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - let key = key_helper + .bucket() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + let key = helper + .key() .get_existing_matching_key(&query.key_pattern) .await?; @@ -376,9 +373,8 @@ impl AdminRpcHandler { let bucket_id = self .garage .bucket_helper() - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; + .admin_get_existing_matching_bucket(&query.bucket) + .await?; let mut bucket = self .garage @@ -418,9 +414,8 @@ impl AdminRpcHandler { let bucket_id = self .garage .bucket_helper() - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; + .admin_get_existing_matching_bucket(&query.bucket) + .await?; let mut bucket = self .garage @@ -477,9 +472,8 @@ impl AdminRpcHandler { bucket_ids.push( self.garage .bucket_helper() - .resolve_global_bucket_name(b) - .await? - .ok_or_bad_request(format!("Bucket not found: {}", b))?, + .admin_get_existing_matching_bucket(b) + .await?, ); } diff --git a/src/garage/admin/key.rs b/src/garage/admin/key.rs index 1c92670c..bd010d2c 100644 --- a/src/garage/admin/key.rs +++ b/src/garage/admin/key.rs @@ -76,9 +76,10 @@ impl AdminRpcHandler { } async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result { - let key_helper = self.garage.key_helper(); + let helper = self.garage.locked_helper().await; - let mut key = key_helper + let mut key = helper + .key() .get_existing_matching_key(&query.key_pattern) .await?; @@ -88,7 +89,7 @@ impl AdminRpcHandler { )); } - key_helper.delete_key(&mut key).await?; + helper.delete_key(&mut key).await?; Ok(AdminRpc::Ok(format!( "Key {} was deleted successfully.", diff --git a/src/garage/admin/mod.rs b/src/garage/admin/mod.rs index b6f9c426..6ae8fa88 100644 --- a/src/garage/admin/mod.rs +++ b/src/garage/admin/mod.rs @@ -4,9 +4,11 @@ mod key; use std::collections::HashMap; use std::fmt::Write; +use std::future::Future; use std::sync::Arc; -use async_trait::async_trait; +use futures::future::FutureExt; + use serde::{Deserialize, Serialize}; use format_table::format_table_to_string; @@ -18,7 +20,7 @@ use garage_util::error::Error as GarageError; use garage_table::replication::*; use garage_table::*; -use garage_rpc::ring::PARTITION_BITS; +use garage_rpc::layout::PARTITION_BITS; use garage_rpc::*; use garage_block::manager::BlockResyncErrorInfo; @@ -27,7 +29,6 @@ use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_model::helper::error::{Error, OkOrBadRequest}; use garage_model::key_table::*; -use garage_model::migrate::Migrate; use garage_model::s3::mpu_table::MultipartUpload; use garage_model::s3::version_table::Version; @@ -42,10 +43,10 @@ pub enum AdminRpc { BucketOperation(BucketOperation), KeyOperation(KeyOperation), LaunchRepair(RepairOpt), - Migrate(MigrateOpt), Stats(StatsOpt), Worker(WorkerOperation), BlockOperation(BlockOperation), + MetaOperation(MetaOperation), // Replies Ok(String), @@ -95,24 +96,6 @@ impl AdminRpcHandler { admin } - // ================ MIGRATION COMMANDS ==================== - - async fn handle_migrate(self: &Arc, opt: MigrateOpt) -> Result { - if !opt.yes { - return Err(Error::BadRequest( - "Please provide the --yes flag to initiate migration operation.".to_string(), - )); - } - - let m = Migrate { - garage: self.garage.clone(), - }; - match opt.what { - MigrateWhat::Buckets050 => m.migrate_buckets050().await, - }?; - Ok(AdminRpc::Ok("Migration successfull.".into())) - } - // ================ REPAIR COMMANDS ==================== async fn handle_launch_repair(self: &Arc, opt: RepairOpt) -> Result { @@ -126,8 +109,8 @@ impl AdminRpcHandler { opt_to_send.all_nodes = false; let mut failures = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { let node = (*node).into(); let resp = self .endpoint @@ -163,9 +146,14 @@ impl AdminRpcHandler { async fn handle_stats(&self, opt: StatsOpt) -> Result { if opt.all_nodes { let mut ret = String::new(); - let ring = self.garage.system.ring.borrow().clone(); + let mut all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in self.garage.system.get_known_nodes().iter() { + if node.is_up && !all_nodes.contains(&node.id) { + all_nodes.push(node.id); + } + } - for node in ring.layout.node_ids().iter() { + for node in all_nodes.iter() { let mut opt = opt.clone(); opt.all_nodes = false; opt.skip_global = true; @@ -217,11 +205,11 @@ impl AdminRpcHandler { // Gather table statistics let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tGcTodo".into()]; - table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?); + table.push(self.gather_table_stats(&self.garage.bucket_table)?); + table.push(self.gather_table_stats(&self.garage.key_table)?); + table.push(self.gather_table_stats(&self.garage.object_table)?); + table.push(self.gather_table_stats(&self.garage.version_table)?); + table.push(self.gather_table_stats(&self.garage.block_ref_table)?); write!( &mut ret, "\nTable stats:\n{}", @@ -231,15 +219,7 @@ impl AdminRpcHandler { // Gather block manager statistics writeln!(&mut ret, "\nBlock manager stats:").unwrap(); - let rc_len = if opt.detailed { - self.garage.block_manager.rc_len()?.to_string() - } else { - self.garage - .block_manager - .rc_fast_len()? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()) - }; + let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string(); writeln!( &mut ret, @@ -250,20 +230,16 @@ impl AdminRpcHandler { writeln!( &mut ret, " resync queue length: {}", - self.garage.block_manager.resync.queue_len()? + self.garage.block_manager.resync.queue_approximate_len()? ) .unwrap(); writeln!( &mut ret, " blocks with resync errors: {}", - self.garage.block_manager.resync.errors_len()? + self.garage.block_manager.resync.errors_approximate_len()? ) .unwrap(); - if !opt.detailed { - writeln!(&mut ret, "\nIf values are missing above (marked as NC), consider adding the --detailed flag (this will be slow).").unwrap(); - } - if !opt.skip_global { write!(&mut ret, "\n{}", self.gather_cluster_stats()).unwrap(); } @@ -274,11 +250,11 @@ impl AdminRpcHandler { fn gather_cluster_stats(&self) -> String { let mut ret = String::new(); - // Gather storage node and free space statistics - let layout = &self.garage.system.ring.borrow().layout; + // Gather storage node and free space statistics for current nodes + let layout = &self.garage.system.cluster_layout(); let mut node_partition_count = HashMap::::new(); - for short_id in layout.ring_assignment_data.iter() { - let id = layout.node_id_vec[*short_id as usize]; + for short_id in layout.current().ring_assignment_data.iter() { + let id = layout.current().node_id_vec[*short_id as usize]; *node_partition_count.entry(id).or_default() += 1; } let node_info = self @@ -293,8 +269,8 @@ impl AdminRpcHandler { for (id, parts) in node_partition_count.iter() { let info = node_info.get(id); let status = info.map(|x| &x.status); - let role = layout.roles.get(id).and_then(|x| x.0.as_ref()); - let hostname = status.map(|x| x.hostname.as_str()).unwrap_or("?"); + let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref()); + let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?"); let zone = role.map(|x| x.zone.as_str()).unwrap_or("?"); let capacity = role .map(|x| x.capacity_string()) @@ -365,42 +341,26 @@ impl AdminRpcHandler { ret } - fn gather_table_stats( - &self, - t: &Arc>, - detailed: bool, - ) -> Result + fn gather_table_stats(&self, t: &Arc>) -> Result where F: TableSchema + 'static, R: TableReplication + 'static, { - let (data_len, mkl_len) = if detailed { - ( - t.data.store.len().map_err(GarageError::from)?.to_string(), - t.merkle_updater.merkle_tree_len()?.to_string(), - ) - } else { - ( - t.data - .store - .fast_len() - .map_err(GarageError::from)? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()), - t.merkle_updater - .merkle_tree_fast_len()? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()), - ) - }; + let data_len = t + .data + .store + .approximate_len() + .map_err(GarageError::from)? + .to_string(); + let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string(); Ok(format!( " {}\t{}\t{}\t{}\t{}", F::TABLE_NAME, data_len, mkl_len, - t.merkle_updater.todo_len()?, - t.data.gc_todo_len()? + t.merkle_updater.todo_approximate_len()?, + t.data.gc_todo_approximate_len()? )) } @@ -440,8 +400,8 @@ impl AdminRpcHandler { ) -> Result { if all_nodes { let mut ret = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { let node = (*node).into(); match self .endpoint @@ -488,8 +448,8 @@ impl AdminRpcHandler { ) -> Result { if all_nodes { let mut ret = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { let node = (*node).into(); match self .endpoint @@ -518,24 +478,68 @@ impl AdminRpcHandler { )])) } } -} -#[async_trait] -impl EndpointHandler for AdminRpcHandler { - async fn handle( - self: &Arc, - message: &AdminRpc, - _from: NodeID, - ) -> Result { - match message { - AdminRpc::BucketOperation(bo) => self.handle_bucket_cmd(bo).await, - AdminRpc::KeyOperation(ko) => self.handle_key_cmd(ko).await, - AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await, - AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await, - AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await, - AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await, - AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await, - m => Err(GarageError::unexpected_rpc_message(m).into()), + // ================ META DB COMMANDS ==================== + + async fn handle_meta_cmd(self: &Arc, mo: &MetaOperation) -> Result { + match mo { + MetaOperation::Snapshot { all: true } => { + let to = self.garage.system.cluster_layout().all_nodes().to_vec(); + + let resps = futures::future::join_all(to.iter().map(|to| async move { + let to = (*to).into(); + self.endpoint + .call( + &to, + AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }), + PRIO_NORMAL, + ) + .await? + })) + .await; + + let mut ret = vec![]; + for (to, resp) in to.iter().zip(resps.iter()) { + let res_str = match resp { + Ok(_) => "ok".to_string(), + Err(e) => format!("error: {}", e), + }; + ret.push(format!("{:?}\t{}", to, res_str)); + } + + if resps.iter().any(Result::is_err) { + Err(GarageError::Message(format_table_to_string(ret)).into()) + } else { + Ok(AdminRpc::Ok(format_table_to_string(ret))) + } + } + MetaOperation::Snapshot { all: false } => { + garage_model::snapshot::async_snapshot_metadata(&self.garage).await?; + Ok(AdminRpc::Ok("Snapshot has been saved.".into())) + } } } } + +impl EndpointHandler for AdminRpcHandler { + fn handle( + self: &Arc, + message: &AdminRpc, + _from: NodeID, + ) -> impl Future> + Send { + let self2 = self.clone(); + async move { + match message { + AdminRpc::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await, + AdminRpc::KeyOperation(ko) => self2.handle_key_cmd(ko).await, + AdminRpc::LaunchRepair(opt) => self2.handle_launch_repair(opt.clone()).await, + AdminRpc::Stats(opt) => self2.handle_stats(opt.clone()).await, + AdminRpc::Worker(wo) => self2.handle_worker_cmd(wo).await, + AdminRpc::BlockOperation(bo) => self2.handle_block_cmd(bo).await, + AdminRpc::MetaOperation(mo) => self2.handle_meta_cmd(mo).await, + m => Err(GarageError::unexpected_rpc_message(m).into()), + } + } + .boxed() + } +} diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index 48359614..44d3d96c 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::time::Duration; use format_table::format_table; @@ -33,9 +33,6 @@ pub async fn cli_command_dispatch( Command::Key(ko) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::KeyOperation(ko)).await } - Command::Migrate(mo) => { - cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Migrate(mo)).await - } Command::Repair(ro) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::LaunchRepair(ro)).await } @@ -44,56 +41,74 @@ pub async fn cli_command_dispatch( Command::Block(bo) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await } + Command::Meta(mo) => { + cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::MetaOperation(mo)).await + } _ => unreachable!(), } } pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> Result<(), Error> { - let status = match rpc_cli - .call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL) - .await?? - { - SystemRpc::ReturnKnownNodes(nodes) => nodes, - resp => return Err(Error::Message(format!("Invalid RPC response: {:?}", resp))), - }; + let status = fetch_status(rpc_cli, rpc_host).await?; let layout = fetch_layout(rpc_cli, rpc_host).await?; println!("==== HEALTHY NODES ===="); let mut healthy_nodes = vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tDataAvail".to_string()]; for adv in status.iter().filter(|adv| adv.is_up) { - match layout.roles.get(&adv.id) { - Some(NodeRoleV(Some(cfg))) => { - let data_avail = match &adv.status.data_disk_avail { - _ if cfg.capacity.is_none() => "N/A".into(), - Some((avail, total)) => { - let pct = (*avail as f64) / (*total as f64) * 100.; - let avail = bytesize::ByteSize::b(*avail); - format!("{} ({:.1}%)", avail, pct) - } - None => "?".into(), - }; + let host = adv.status.hostname.as_deref().unwrap_or("?"); + let addr = match adv.addr { + Some(addr) => addr.to_string(), + None => "N/A".to_string(), + }; + if let Some(NodeRoleV(Some(cfg))) = layout.current().roles.get(&adv.id) { + let data_avail = match &adv.status.data_disk_avail { + _ if cfg.capacity.is_none() => "N/A".into(), + Some((avail, total)) => { + let pct = (*avail as f64) / (*total as f64) * 100.; + let avail = bytesize::ByteSize::b(*avail); + format!("{} ({:.1}%)", avail, pct) + } + None => "?".into(), + }; + healthy_nodes.push(format!( + "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}", + id = adv.id, + host = host, + addr = addr, + tags = cfg.tags.join(","), + zone = cfg.zone, + capacity = cfg.capacity_string(), + data_avail = data_avail, + )); + } else { + let prev_role = layout + .versions + .iter() + .rev() + .find_map(|x| match x.roles.get(&adv.id) { + Some(NodeRoleV(Some(cfg))) => Some(cfg), + _ => None, + }); + if let Some(cfg) = prev_role { healthy_nodes.push(format!( - "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}", + "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\tdraining metadata...", id = adv.id, - host = adv.status.hostname, - addr = adv.addr, + host = host, + addr = addr, tags = cfg.tags.join(","), zone = cfg.zone, - capacity = cfg.capacity_string(), - data_avail = data_avail, )); - } - _ => { - let new_role = match layout.staging_roles.get(&adv.id) { - Some(NodeRoleV(Some(_))) => "(pending)", + } else { + let new_role = match layout.staging.get().roles.get(&adv.id) { + Some(NodeRoleV(Some(_))) => "pending...", _ => "NO ROLE ASSIGNED", }; healthy_nodes.push(format!( - "{id:?}\t{h}\t{addr}\t{new_role}", + "{id:?}\t{h}\t{addr}\t\t\t{new_role}", id = adv.id, - h = adv.status.hostname, - addr = adv.addr, + h = host, + addr = addr, new_role = new_role, )); } @@ -101,51 +116,73 @@ pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> } format_table(healthy_nodes); - let status_keys = status.iter().map(|adv| adv.id).collect::>(); - let failure_case_1 = status + // Determine which nodes are unhealthy and print that to stdout + let status_map = status .iter() - .any(|adv| !adv.is_up && matches!(layout.roles.get(&adv.id), Some(NodeRoleV(Some(_))))); - let failure_case_2 = layout - .roles - .items() - .iter() - .any(|(id, _, v)| !status_keys.contains(id) && v.0.is_some()); - if failure_case_1 || failure_case_2 { - println!("\n==== FAILED NODES ===="); - let mut failed_nodes = - vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tLast seen".to_string()]; - for adv in status.iter().filter(|adv| !adv.is_up) { - if let Some(NodeRoleV(Some(cfg))) = layout.roles.get(&adv.id) { - let tf = timeago::Formatter::new(); - failed_nodes.push(format!( - "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}", - id = adv.id, - host = adv.status.hostname, - addr = adv.addr, - tags = cfg.tags.join(","), - zone = cfg.zone, - capacity = cfg.capacity_string(), - last_seen = adv - .last_seen_secs_ago + .map(|adv| (adv.id, adv)) + .collect::>(); + + let tf = timeago::Formatter::new(); + let mut drain_msg = false; + let mut failed_nodes = vec!["ID\tHostname\tTags\tZone\tCapacity\tLast seen".to_string()]; + let mut listed = HashSet::new(); + for ver in layout.versions.iter().rev() { + for (node, _, role) in ver.roles.items().iter() { + let cfg = match role { + NodeRoleV(Some(role)) if role.capacity.is_some() => role, + _ => continue, + }; + + if listed.contains(node) { + continue; + } + listed.insert(*node); + + let adv = status_map.get(node); + if adv.map(|x| x.is_up).unwrap_or(false) { + continue; + } + + // Node is in a layout version, is not a gateway node, and is not up: + // it is in a failed state, add proper line to the output + let (host, last_seen) = match adv { + Some(adv) => ( + adv.status.hostname.as_deref().unwrap_or("?"), + adv.last_seen_secs_ago .map(|s| tf.convert(Duration::from_secs(s))) .unwrap_or_else(|| "never seen".into()), - )); - } - } - for (id, _, role_v) in layout.roles.items().iter() { - if let NodeRoleV(Some(cfg)) = role_v { - if !status_keys.contains(id) { - failed_nodes.push(format!( - "{id:?}\t??\t??\t[{tags}]\t{zone}\t{capacity}\tnever seen", - id = id, - tags = cfg.tags.join(","), - zone = cfg.zone, - capacity = cfg.capacity_string(), - )); - } - } + ), + None => ("??", "never seen".into()), + }; + let capacity = if ver.version == layout.current().version { + cfg.capacity_string() + } else { + drain_msg = true; + "draining metadata...".to_string() + }; + failed_nodes.push(format!( + "{id:?}\t{host}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}", + id = node, + host = host, + tags = cfg.tags.join(","), + zone = cfg.zone, + capacity = capacity, + last_seen = last_seen, + )); } + } + + if failed_nodes.len() > 1 { + println!("\n==== FAILED NODES ===="); format_table(failed_nodes); + if drain_msg { + println!(); + println!("Your cluster is expecting to drain data from nodes that are currently unavailable."); + println!("If these nodes are definitely dead, please review the layout history with"); + println!( + "`garage layout history` and use `garage layout skip-dead-nodes` to force progress." + ); + } } if print_staging_role_changes(&layout) { @@ -226,3 +263,18 @@ pub async fn cmd_admin( } Ok(()) } + +// ---- utility ---- + +pub async fn fetch_status( + rpc_cli: &Endpoint, + rpc_host: NodeID, +) -> Result, Error> { + match rpc_cli + .call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL) + .await?? + { + SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes), + resp => Err(Error::unexpected_rpc_message(resp)), + } +} diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs new file mode 100644 index 00000000..a40fb61f --- /dev/null +++ b/src/garage/cli/convert_db.rs @@ -0,0 +1,65 @@ +use std::path::PathBuf; + +use structopt::StructOpt; + +use garage_db::*; + +/// K2V command line interface +#[derive(StructOpt, Debug)] +pub struct ConvertDbOpt { + /// Input database path (not the same as metadata_dir, see + /// https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0) + #[structopt(short = "i")] + input_path: PathBuf, + /// Input database engine (lmdb or sqlite; limited by db engines + /// enabled in this build) + #[structopt(short = "a")] + input_engine: Engine, + + /// Output database path + #[structopt(short = "o")] + output_path: PathBuf, + /// Output database engine + #[structopt(short = "b")] + output_engine: Engine, + + #[structopt(flatten)] + #[allow(dead_code)] + db_open: OpenDbOpt, +} + +/// Overrides for database open operation +#[derive(StructOpt, Debug, Default)] +pub struct OpenDbOpt { + #[cfg(feature = "lmdb")] + #[structopt(flatten)] + lmdb: OpenLmdbOpt, +} + +/// Overrides for LMDB database open operation +#[cfg(feature = "lmdb")] +#[derive(StructOpt, Debug, Default)] +pub struct OpenLmdbOpt { + /// LMDB map size override + /// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB) + #[cfg(feature = "lmdb")] + #[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)] + map_size: Option, +} + +pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { + if args.input_engine == args.output_engine { + return Err(Error("input and output database engine must differ".into())); + } + + let opt = OpenOpt { + #[cfg(feature = "lmdb")] + lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize), + ..Default::default() + }; + + let input = open_db(&args.input_path, args.input_engine, &opt)?; + let output = open_db(&args.output_path, args.output_engine, &opt)?; + output.import(&input)?; + Ok(()) +} diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs index 20813f1c..43ca5c09 100644 --- a/src/garage/cli/init.rs +++ b/src/garage/cli/init.rs @@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { idstr ); eprintln!( - "where is their own node identifier in the format: @:" + "where is their own node identifier in the format: @:" ); eprintln!(); eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:"); diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs index ce2b11e0..f053eef4 100644 --- a/src/garage/cli/layout.rs +++ b/src/garage/cli/layout.rs @@ -32,6 +32,10 @@ pub async fn cli_layout_command_dispatch( LayoutOperation::Config(config_opt) => { cmd_config_layout(system_rpc_endpoint, rpc_host, config_opt).await } + LayoutOperation::History => cmd_layout_history(system_rpc_endpoint, rpc_host).await, + LayoutOperation::SkipDeadNodes(assume_sync_opt) => { + cmd_layout_skip_dead_nodes(system_rpc_endpoint, rpc_host, assume_sync_opt).await + } } } @@ -49,6 +53,7 @@ pub async fn cmd_assign_role( }; let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + let all_nodes = layout.get_all_nodes(); let added_nodes = args .node_ids @@ -58,21 +63,23 @@ pub async fn cmd_assign_role( status .iter() .map(|adv| adv.id) - .chain(layout.node_ids().iter().cloned()), + .chain(all_nodes.iter().cloned()), node_id, ) }) .collect::, _>>()?; - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging_roles); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); for replaced in args.replace.iter() { - let replaced_node = find_matching_node(layout.node_ids().iter().cloned(), replaced)?; + let replaced_node = find_matching_node(all_nodes.iter().cloned(), replaced)?; match roles.get(&replaced_node) { Some(NodeRoleV(Some(_))) => { layout - .staging_roles + .staging + .get_mut() + .roles .merge(&roles.update_mutator(replaced_node, NodeRoleV(None))); } _ => { @@ -122,7 +129,7 @@ pub async fn cmd_assign_role( zone: args .zone .clone() - .ok_or("Please specifiy a zone with the -z flag")?, + .ok_or("Please specify a zone with the -z flag")?, capacity, tags: args.tags.clone(), } @@ -130,13 +137,15 @@ pub async fn cmd_assign_role( }; layout - .staging_roles + .staging + .get_mut() + .roles .merge(&roles.update_mutator(added_node, NodeRoleV(Some(new_entry)))); } send_layout(rpc_cli, rpc_host, layout).await?; - println!("Role changes are staged but not yet commited."); + println!("Role changes are staged but not yet committed."); println!("Use `garage layout show` to view staged role changes,"); println!("and `garage layout apply` to enact staged changes."); Ok(()) @@ -149,19 +158,21 @@ pub async fn cmd_remove_role( ) -> Result<(), Error> { let mut layout = fetch_layout(rpc_cli, rpc_host).await?; - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging_roles); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); let deleted_node = find_matching_node(roles.items().iter().map(|(id, _, _)| *id), &args.node_id)?; layout - .staging_roles + .staging + .get_mut() + .roles .merge(&roles.update_mutator(deleted_node, NodeRoleV(None))); send_layout(rpc_cli, rpc_host, layout).await?; - println!("Role removal is staged but not yet commited."); + println!("Role removal is staged but not yet committed."); println!("Use `garage layout show` to view staged role changes,"); println!("and `garage layout apply` to enact staged changes."); Ok(()) @@ -174,13 +185,16 @@ pub async fn cmd_show_layout( let layout = fetch_layout(rpc_cli, rpc_host).await?; println!("==== CURRENT CLUSTER LAYOUT ===="); - print_cluster_layout(&layout, "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes."); + print_cluster_layout(layout.current(), "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes."); println!(); - println!("Current cluster layout version: {}", layout.version); + println!( + "Current cluster layout version: {}", + layout.current().version + ); let has_role_changes = print_staging_role_changes(&layout); if has_role_changes { - let v = layout.version; + let v = layout.current().version; let res_apply = layout.apply_staged_changes(Some(v + 1)); // this will print the stats of what partitions @@ -189,7 +203,7 @@ pub async fn cmd_show_layout( Ok((layout, msg)) => { println!(); println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ===="); - print_cluster_layout(&layout, "No nodes have a role in the new layout."); + print_cluster_layout(layout.current(), "No nodes have a role in the new layout."); println!(); for line in msg.iter() { @@ -199,16 +213,12 @@ pub async fn cmd_show_layout( println!(); println!(" garage layout apply --version {}", v + 1); println!(); - println!( - "You can also revert all proposed changes with: garage layout revert --version {}", - v + 1) + println!("You can also revert all proposed changes with: garage layout revert"); } Err(e) => { println!("Error while trying to compute the assignment: {}", e); println!("This new layout cannot yet be applied."); - println!( - "You can also revert all proposed changes with: garage layout revert --version {}", - v + 1) + println!("You can also revert all proposed changes with: garage layout revert"); } } } @@ -241,9 +251,15 @@ pub async fn cmd_revert_layout( rpc_host: NodeID, revert_opt: RevertLayoutOpt, ) -> Result<(), Error> { + if !revert_opt.yes { + return Err(Error::Message( + "Please add the --yes flag to run the layout revert operation".into(), + )); + } + let layout = fetch_layout(rpc_cli, rpc_host).await?; - let layout = layout.revert_staged_changes(revert_opt.version)?; + let layout = layout.revert_staged_changes()?; send_layout(rpc_cli, rpc_host, layout).await?; @@ -266,11 +282,11 @@ pub async fn cmd_config_layout( .parse::() .ok_or_message("invalid zone redundancy value")?; if let ZoneRedundancy::AtLeast(r_int) = r { - if r_int > layout.replication_factor { + if r_int > layout.current().replication_factor { return Err(Error::Message(format!( "The zone redundancy must be smaller or equal to the \ replication factor ({}).", - layout.replication_factor + layout.current().replication_factor ))); } else if r_int < 1 { return Err(Error::Message( @@ -280,7 +296,9 @@ pub async fn cmd_config_layout( } layout - .staging_parameters + .staging + .get_mut() + .parameters .update(LayoutParameters { zone_redundancy: r }); println!("The zone redundancy parameter has been set to '{}'.", r); did_something = true; @@ -297,25 +315,178 @@ pub async fn cmd_config_layout( Ok(()) } +pub async fn cmd_layout_history( + rpc_cli: &Endpoint, + rpc_host: NodeID, +) -> Result<(), Error> { + let layout = fetch_layout(rpc_cli, rpc_host).await?; + let min_stored = layout.min_stored(); + + println!("==== LAYOUT HISTORY ===="); + let mut table = vec!["Version\tStatus\tStorage nodes\tGateway nodes".to_string()]; + for ver in layout + .versions + .iter() + .rev() + .chain(layout.old_versions.iter().rev()) + { + let status = if ver.version == layout.current().version { + "current" + } else if ver.version >= min_stored { + "draining" + } else { + "historical" + }; + table.push(format!( + "#{}\t{}\t{}\t{}", + ver.version, + status, + ver.roles + .items() + .iter() + .filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_some())) + .count(), + ver.roles + .items() + .iter() + .filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_none())) + .count(), + )); + } + format_table(table); + println!(); + + if layout.versions.len() > 1 { + println!("==== UPDATE TRACKERS ===="); + println!("Several layout versions are currently live in the cluster, and data is being migrated."); + println!( + "This is the internal data that Garage stores to know which nodes have what data." + ); + println!(); + let mut table = vec!["Node\tAck\tSync\tSync_ack".to_string()]; + let all_nodes = layout.get_all_nodes(); + for node in all_nodes.iter() { + table.push(format!( + "{:?}\t#{}\t#{}\t#{}", + node, + layout.update_trackers.ack_map.get(node, min_stored), + layout.update_trackers.sync_map.get(node, min_stored), + layout.update_trackers.sync_ack_map.get(node, min_stored), + )); + } + table[1..].sort(); + format_table(table); + + let min_ack = layout + .update_trackers + .ack_map + .min_among(&all_nodes, layout.min_stored()); + + println!(); + println!( + "If some nodes are not catching up to the latest layout version in the update trackers," + ); + println!("it might be because they are offline or unable to complete a sync successfully."); + if min_ack < layout.current().version { + println!( + "You may force progress using `garage layout skip-dead-nodes --version {}`", + layout.current().version + ); + } else { + println!( + "You may force progress using `garage layout skip-dead-nodes --version {} --allow-missing-data`.", + layout.current().version + ); + } + } else { + println!("Your cluster is currently in a stable state with a single live layout version."); + println!("No metadata migration is in progress. Note that the migration of data blocks is not tracked,"); + println!( + "so you might want to keep old nodes online until their data directories become empty." + ); + } + + Ok(()) +} + +pub async fn cmd_layout_skip_dead_nodes( + rpc_cli: &Endpoint, + rpc_host: NodeID, + opt: SkipDeadNodesOpt, +) -> Result<(), Error> { + let status = fetch_status(rpc_cli, rpc_host).await?; + let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + + if layout.versions.len() == 1 { + return Err(Error::Message( + "This command cannot be called when there is only one live cluster layout version" + .into(), + )); + } + + let min_v = layout.min_stored(); + if opt.version <= min_v || opt.version > layout.current().version { + return Err(Error::Message(format!( + "Invalid version, you may use the following version numbers: {}", + (min_v + 1..=layout.current().version) + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ))); + } + + let all_nodes = layout.get_all_nodes(); + let mut did_something = false; + for node in all_nodes.iter() { + // Update ACK tracker for dead nodes or for all nodes if --allow-missing-data + if opt.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) { + if layout.update_trackers.ack_map.set_max(*node, opt.version) { + println!("Increased the ACK tracker for node {:?}", node); + did_something = true; + } + } + + // If --allow-missing-data, update SYNC tracker for all nodes. + if opt.allow_missing_data { + if layout.update_trackers.sync_map.set_max(*node, opt.version) { + println!("Increased the SYNC tracker for node {:?}", node); + did_something = true; + } + } + } + + if did_something { + send_layout(rpc_cli, rpc_host, layout).await?; + println!("Success."); + Ok(()) + } else if !opt.allow_missing_data { + Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into())) + } else { + Err(Error::Message( + "Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(), + )) + } +} + // --- utility --- pub async fn fetch_layout( rpc_cli: &Endpoint, rpc_host: NodeID, -) -> Result { +) -> Result { match rpc_cli .call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL) .await?? { SystemRpc::AdvertiseClusterLayout(t) => Ok(t), - resp => Err(Error::Message(format!("Invalid RPC response: {:?}", resp))), + resp => Err(Error::unexpected_rpc_message(resp)), } } pub async fn send_layout( rpc_cli: &Endpoint, rpc_host: NodeID, - layout: ClusterLayout, + layout: LayoutHistory, ) -> Result<(), Error> { rpc_cli .call( @@ -327,7 +498,7 @@ pub async fn send_layout( Ok(()) } -pub fn print_cluster_layout(layout: &ClusterLayout, empty_msg: &str) { +pub fn print_cluster_layout(layout: &LayoutVersion, empty_msg: &str) { let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()]; for (id, _, role) in layout.roles.items().iter() { let role = match &role.0 { @@ -366,21 +537,22 @@ pub fn print_cluster_layout(layout: &ClusterLayout, empty_msg: &str) { } } -pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool { - let has_role_changes = layout - .staging_roles +pub fn print_staging_role_changes(layout: &LayoutHistory) -> bool { + let staging = layout.staging.get(); + let has_role_changes = staging + .roles .items() .iter() - .any(|(k, _, v)| layout.roles.get(k) != Some(v)); - let has_layout_changes = *layout.staging_parameters.get() != layout.parameters; + .any(|(k, _, v)| layout.current().roles.get(k) != Some(v)); + let has_layout_changes = *staging.parameters.get() != layout.current().parameters; if has_role_changes || has_layout_changes { println!(); println!("==== STAGED ROLE CHANGES ===="); if has_role_changes { let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()]; - for (id, _, role) in layout.staging_roles.items().iter() { - if layout.roles.get(id) == Some(role) { + for (id, _, role) in staging.roles.items().iter() { + if layout.current().roles.get(id) == Some(role) { continue; } if let Some(role) = &role.0 { @@ -402,7 +574,7 @@ pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool { if has_layout_changes { println!( "Zone redundancy: {}", - layout.staging_parameters.get().zone_redundancy + staging.parameters.get().zone_redundancy ); } true diff --git a/src/garage/cli/mod.rs b/src/garage/cli/mod.rs index 17a2d8ce..e131f62c 100644 --- a/src/garage/cli/mod.rs +++ b/src/garage/cli/mod.rs @@ -4,6 +4,8 @@ pub(crate) mod layout; pub(crate) mod structs; pub(crate) mod util; +pub(crate) mod convert_db; + pub(crate) use cmd::*; pub(crate) use init::*; pub(crate) use layout::*; diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index c4ebeb1a..386a213b 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -3,6 +3,8 @@ use structopt::StructOpt; use garage_util::version::garage_version; +use crate::cli::convert_db; + #[derive(StructOpt, Debug)] pub enum Command { /// Run Garage server @@ -29,11 +31,6 @@ pub enum Command { #[structopt(name = "key", version = garage_version())] Key(KeyOperation), - /// Run migrations from previous Garage version - /// (DO NOT USE WITHOUT READING FULL DOCUMENTATION) - #[structopt(name = "migrate", version = garage_version())] - Migrate(MigrateOpt), - /// Start repair of node data on remote node #[structopt(name = "repair", version = garage_version())] Repair(RepairOpt), @@ -51,14 +48,23 @@ pub enum Command { #[structopt(name = "worker", version = garage_version())] Worker(WorkerOperation), - /// Low-level debug operations on data blocks + /// Low-level node-local debug operations on data blocks #[structopt(name = "block", version = garage_version())] Block(BlockOperation), + + /// Operations on the metadata db + #[structopt(name = "meta", version = garage_version())] + Meta(MetaOperation), + + /// Convert metadata db between database engine formats + #[structopt(name = "convert-db", version = garage_version())] + ConvertDb(convert_db::ConvertDbOpt), } #[derive(StructOpt, Debug)] pub enum NodeOperation { - /// Print identifier (public key) of this Garage node + /// Print the full node ID (public key) of this Garage node, and its publicly reachable IP + /// address and port if they are specified in config file under `rpc_public_addr` #[structopt(name = "id", version = garage_version())] NodeId(NodeIdOpt), @@ -76,8 +82,9 @@ pub struct NodeIdOpt { #[derive(StructOpt, Debug)] pub struct ConnectNodeOpt { - /// Node public key and address, in the format: - /// `@:` + /// Full node ID (public key) and IP address and port, in the format: + /// `@:`. + /// You can retrieve this information on the target node using `garage node id`. pub(crate) node: String, } @@ -106,6 +113,14 @@ pub enum LayoutOperation { /// Revert staged changes to cluster layout #[structopt(name = "revert", version = garage_version())] Revert(RevertLayoutOpt), + + /// View the history of layouts in the cluster + #[structopt(name = "history", version = garage_version())] + History, + + /// Skip dead nodes when awaiting for a new layout version to be synchronized + #[structopt(name = "skip-dead-nodes", version = garage_version())] + SkipDeadNodes(SkipDeadNodesOpt), } #[derive(StructOpt, Debug)] @@ -158,9 +173,21 @@ pub struct ApplyLayoutOpt { #[derive(StructOpt, Debug)] pub struct RevertLayoutOpt { - /// Version number of old configuration to which to revert + /// The revert operation will not be ran unless this flag is added + #[structopt(long = "yes")] + pub(crate) yes: bool, +} + +#[derive(StructOpt, Debug)] +pub struct SkipDeadNodesOpt { + /// Version number of the layout to assume is currently up-to-date. + /// This will generally be the current layout version. #[structopt(long = "version")] - pub(crate) version: Option, + pub(crate) version: u64, + /// Allow the skip even if a quorum of nodes could not be found for + /// the data among the remaining nodes + #[structopt(long = "allow-missing-data")] + pub(crate) allow_missing_data: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug)] @@ -417,23 +444,6 @@ pub struct KeyImportOpt { pub yes: bool, } -#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] -pub struct MigrateOpt { - /// Confirm the launch of the migrate operation - #[structopt(long = "yes")] - pub yes: bool, - - #[structopt(subcommand)] - pub what: MigrateWhat, -} - -#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] -pub enum MigrateWhat { - /// Migrate buckets and permissions from v0.5.0 - #[structopt(name = "buckets050", version = garage_version())] - Buckets050, -} - #[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] pub struct RepairOpt { /// Launch repair operation on all nodes @@ -453,9 +463,13 @@ pub enum RepairWhat { /// Do a full sync of metadata tables #[structopt(name = "tables", version = garage_version())] Tables, - /// Repair (resync/rebalance) the set of stored blocks + /// Repair (resync/rebalance) the set of stored blocks in the cluster #[structopt(name = "blocks", version = garage_version())] Blocks, + /// Clear the block resync queue. The list of blocks in errored state + /// is cleared as well. You MUST run `garage repair blocks` after invoking this. + #[structopt(name = "clear-resync-queue", version = garage_version())] + ClearResyncQueue, /// Repropagate object deletions to the version table #[structopt(name = "versions", version = garage_version())] Versions, @@ -463,15 +477,21 @@ pub enum RepairWhat { #[structopt(name = "mpu", version = garage_version())] MultipartUploads, /// Repropagate version deletions to the block ref table - #[structopt(name = "block_refs", version = garage_version())] + #[structopt(name = "block-refs", version = garage_version())] BlockRefs, + /// Recalculate block reference counters + #[structopt(name = "block-rc", version = garage_version())] + BlockRc, + /// Fix inconsistency in bucket aliases (WARNING: EXPERIMENTAL) + #[structopt(name = "aliases", version = garage_version())] + Aliases, /// Verify integrity of all blocks on disc #[structopt(name = "scrub", version = garage_version())] Scrub { #[structopt(subcommand)] cmd: ScrubCmd, }, - /// Rebalance data blocks among storage locations + /// Rebalance data blocks among HDDs on individual nodes #[structopt(name = "rebalance", version = garage_version())] Rebalance, } @@ -525,10 +545,6 @@ pub struct StatsOpt { #[structopt(short = "a", long = "all-nodes")] pub all_nodes: bool, - /// Gather detailed statistics (this can be long) - #[structopt(short = "d", long = "detailed")] - pub detailed: bool, - /// Don't show global cluster stats (internal use in RPC) #[structopt(skip)] #[serde(default)] @@ -609,3 +625,14 @@ pub enum BlockOperation { blocks: Vec, }, } + +#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)] +pub enum MetaOperation { + /// Save a snapshot of the metadata db file + #[structopt(name = "snapshot", version = garage_version())] + Snapshot { + /// Run on all nodes instead of only local node + #[structopt(long = "all")] + all: bool, + }, +} diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs index 2232d395..21c14f42 100644 --- a/src/garage/cli/util.rs +++ b/src/garage/cli/util.rs @@ -450,6 +450,8 @@ pub fn print_block_info( if refcount != nondeleted_count { println!(); - println!("Warning: refcount does not match number of non-deleted versions"); + println!( + "Warning: refcount does not match number of non-deleted versions, you should try `garage repair block-rc`." + ); } } diff --git a/src/garage/main.rs b/src/garage/main.rs index 3d07208c..2703bedd 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -7,6 +7,7 @@ extern crate tracing; mod admin; mod cli; mod repair; +mod secrets; mod server; #[cfg(feature = "telemetry-otlp")] mod tracing_setup; @@ -17,18 +18,17 @@ compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled #[cfg(all(feature = "bundled-libs", feature = "system-libs"))] compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled"); -#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))] -compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite."); +#[cfg(not(any(feature = "lmdb", feature = "sqlite")))] +compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb or sqlite."); use std::net::SocketAddr; use std::path::PathBuf; use structopt::StructOpt; -use netapp::util::parse_and_resolve_peer_addr; -use netapp::NetworkKey; +use garage_net::util::parse_and_resolve_peer_addr; +use garage_net::NetworkKey; -use garage_util::config::Config; use garage_util::error::*; use garage_rpc::system::*; @@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError; use admin::*; use cli::*; +use secrets::Secrets; #[derive(StructOpt, Debug)] #[structopt( @@ -45,8 +46,7 @@ use cli::*; about = "S3-compatible object store for self-hosted geo-distributed deployments" )] struct Opt { - /// Host to connect to for admin operations, in the format: - /// @: + /// Host to connect to for admin operations, in the format: @: #[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")] pub rpc_host: Option, @@ -66,32 +66,12 @@ struct Opt { cmd: Command, } -#[derive(StructOpt, Debug)] -pub struct Secrets { - /// RPC secret network key, used to replace rpc_secret in config.toml when running the - /// daemon or doing admin operations - #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] - pub rpc_secret: Option, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] - pub admin_token: Option, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] - pub metrics_token: Option, -} - #[tokio::main] async fn main() { // Initialize version and features info let features = &[ #[cfg(feature = "k2v")] "k2v", - #[cfg(feature = "sled")] - "sled", #[cfg(feature = "lmdb")] "lmdb", #[cfg(feature = "sqlite")] @@ -127,7 +107,7 @@ async fn main() { ); // Initialize panic handler that aborts on panic and shows a nice message. - // By default, Tokio continues runing normally when a task panics. We want + // By default, Tokio continues running normally when a task panics. We want // to avoid this behavior in Garage as this would risk putting the process in an // unknown/uncontrollable state. We prefer to exit the process and restart it // from scratch, so that it boots back into a fresh, known state. @@ -158,17 +138,8 @@ async fn main() { let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); // Initialize logging as well as other libraries used in Garage - if std::env::var("RUST_LOG").is_err() { - let default_log = match &opt.cmd { - Command::Server => "netapp=info,garage=info", - _ => "netapp=warn,garage=warn", - }; - std::env::set_var("RUST_LOG", default_log) - } - tracing_subscriber::fmt() - .with_writer(std::io::stderr) - .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) - .init(); + init_logging(&opt); + sodiumoxide::init().expect("Unable to init sodiumoxide"); let res = match opt.cmd { @@ -176,6 +147,9 @@ async fn main() { Command::OfflineRepair(repair_opt) => { repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt).await } + Command::ConvertDb(conv_opt) => { + cli::convert_db::do_conversion(conv_opt).map_err(From::from) + } Command::Node(NodeOperation::NodeId(node_id_opt)) => { node_id_command(opt.config_file, node_id_opt.quiet) } @@ -188,8 +162,99 @@ async fn main() { } } +fn init_logging(opt: &Opt) { + if std::env::var("RUST_LOG").is_err() { + let default_log = match &opt.cmd { + Command::Server => "netapp=info,garage=info", + _ => "netapp=warn,garage=warn", + }; + std::env::set_var("RUST_LOG", default_log) + } + + let env_filter = tracing_subscriber::filter::EnvFilter::from_default_env(); + + if std::env::var("GARAGE_LOG_TO_SYSLOG") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "syslog")] + { + use std::ffi::CStr; + use syslog_tracing::{Facility, Options, Syslog}; + + let syslog = Syslog::new( + CStr::from_bytes_with_nul(b"garage\0").unwrap(), + Options::LOG_PID | Options::LOG_PERROR, + Facility::Daemon, + ) + .expect("Unable to init syslog"); + + tracing_subscriber::fmt() + .with_writer(syslog) + .with_env_filter(env_filter) + .with_ansi(false) // disable ANSI escape sequences (colours) + .with_file(false) + .with_level(false) + .without_time() + .compact() + .init(); + + return; + } + #[cfg(not(feature = "syslog"))] + { + eprintln!("Syslog support is not enabled in this build."); + std::process::exit(1); + } + } + + if std::env::var("GARAGE_LOG_TO_JOURNALD") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "journald")] + { + use tracing_journald::{Priority, PriorityMappings}; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + + let registry = tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::sink)) + .with(env_filter); + match tracing_journald::layer() { + Ok(layer) => { + registry + .with(layer.with_priority_mappings(PriorityMappings { + info: Priority::Informational, + debug: Priority::Debug, + ..PriorityMappings::new() + })) + .init(); + } + Err(e) => { + eprintln!("Couldn't connect to journald: {}.", e); + std::process::exit(1); + } + } + return; + } + #[cfg(not(feature = "journald"))] + { + eprintln!("Journald support is not enabled in this build."); + std::process::exit(1); + } + } + + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(env_filter) + .init(); +} + async fn cli_command(opt: Opt) -> Result<(), Error> { - let config = if opt.secrets.rpc_secret.is_none() || opt.rpc_host.is_none() { + let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none()) + || opt.rpc_host.is_none() + { Some(garage_util::config::read_config(opt.config_file.clone()) .err_context(format!("Unable to read configuration file {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy()))?) } else { @@ -197,25 +262,30 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { }; // Find and parse network RPC secret - let net_key_hex_str = opt - .secrets - .rpc_secret - .as_ref() - .or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref())) - .ok_or("No RPC secret provided")?; + let mut rpc_secret = config.as_ref().and_then(|c| c.rpc_secret.clone()); + secrets::fill_secret( + &mut rpc_secret, + &config.as_ref().and_then(|c| c.rpc_secret_file.clone()), + &opt.secrets.rpc_secret, + &opt.secrets.rpc_secret_file, + "rpc_secret", + true, + )?; + + let net_key_hex_str = rpc_secret.ok_or("No RPC secret provided")?; let network_key = NetworkKey::from_slice( - &hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], + &hex::decode(&net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], ) .ok_or("Invalid RPC secret provided (wrong length)")?; // Generate a temporary keypair for our RPC client let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); - let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None); // Find and parse the address of the target host let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host { - let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; + let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; (id, addrs[0], false) } else { let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir) @@ -245,7 +315,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { addr ); } - Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?; + Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?; } let system_rpc_endpoint = netapp.endpoint::(SYSTEM_RPC_PATH.into()); @@ -258,16 +328,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { Ok(x) => Ok(x), } } - -fn fill_secrets(mut config: Config, secrets: Secrets) -> Config { - if secrets.rpc_secret.is_some() { - config.rpc_secret = secrets.rpc_secret; - } - if secrets.admin_token.is_some() { - config.admin.admin_token = secrets.admin_token; - } - if secrets.metrics_token.is_some() { - config.admin.metrics_token = secrets.metrics_token; - } - config -} diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs index f4edcf03..45024e71 100644 --- a/src/garage/repair/offline.rs +++ b/src/garage/repair/offline.rs @@ -6,7 +6,7 @@ use garage_util::error::*; use garage_model::garage::Garage; use crate::cli::structs::*; -use crate::{fill_secrets, Secrets}; +use crate::secrets::{fill_secrets, Secrets}; pub async fn offline_repair( config_file: PathBuf, @@ -20,7 +20,7 @@ pub async fn offline_repair( } info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; info!("Initializing Garage main data store..."); let garage = Garage::new(config)?; diff --git a/src/garage/repair/online.rs b/src/garage/repair/online.rs index 9e4de873..6a7dafcf 100644 --- a/src/garage/repair/online.rs +++ b/src/garage/repair/online.rs @@ -1,9 +1,11 @@ +use std::future::Future; use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; use tokio::sync::watch; +use garage_block::manager::BlockManager; use garage_block::repair::ScrubWorkerCommand; use garage_model::garage::Garage; @@ -16,11 +18,14 @@ use garage_table::replication::*; use garage_table::*; use garage_util::background::*; +use garage_util::data::*; use garage_util::error::Error; use garage_util::migrate::Migrate; use crate::*; +const RC_REPAIR_ITER_COUNT: usize = 64; + pub async fn launch_online_repair( garage: &Arc, bg: &BackgroundRunner, @@ -47,6 +52,13 @@ pub async fn launch_online_repair( info!("Repairing the block refs table"); bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs)); } + RepairWhat::BlockRc => { + info!("Repairing the block reference counters"); + bg.spawn_worker(BlockRcRepair::new( + garage.block_manager.clone(), + garage.block_ref_table.clone(), + )); + } RepairWhat::Blocks => { info!("Repairing the stored blocks"); bg.spawn_worker(garage_block::repair::RepairWorker::new( @@ -76,23 +88,31 @@ pub async fn launch_online_repair( garage.block_manager.clone(), )); } + RepairWhat::Aliases => { + info!("Repairing bucket aliases (foreground)"); + garage.locked_helper().await.repair_aliases().await?; + } + RepairWhat::ClearResyncQueue => { + let garage = garage.clone(); + tokio::task::spawn_blocking(move || garage.block_manager.resync.clear_resync_queue()) + .await?? + } } Ok(()) } // ---- -#[async_trait] trait TableRepair: Send + Sync + 'static { type T: TableSchema; fn table(garage: &Garage) -> &Table; - async fn process( + fn process( &mut self, garage: &Garage, entry: <::T as TableSchema>::E, - ) -> Result; + ) -> impl Future> + Send; } struct TableRepairWorker { @@ -163,7 +183,6 @@ impl Worker for TableRepairWorker { struct RepairVersions; -#[async_trait] impl TableRepair for RepairVersions { type T = VersionTable; @@ -210,7 +229,6 @@ impl TableRepair for RepairVersions { struct RepairBlockRefs; -#[async_trait] impl TableRepair for RepairBlockRefs { type T = BlockRefTable; @@ -246,7 +264,6 @@ impl TableRepair for RepairBlockRefs { struct RepairMpu; -#[async_trait] impl TableRepair for RepairMpu { type T = MultipartUploadTable; @@ -282,3 +299,98 @@ impl TableRepair for RepairMpu { Ok(false) } } + +// ===== block reference counter repair ===== + +pub struct BlockRcRepair { + block_manager: Arc, + block_ref_table: Arc>, + cursor: Hash, + counter: u64, + repairs: u64, +} + +impl BlockRcRepair { + fn new( + block_manager: Arc, + block_ref_table: Arc>, + ) -> Self { + Self { + block_manager, + block_ref_table, + cursor: [0u8; 32].into(), + counter: 0, + repairs: 0, + } + } +} + +#[async_trait] +impl Worker for BlockRcRepair { + fn name(&self) -> String { + format!("Block refcount repair worker") + } + + fn status(&self) -> WorkerStatus { + WorkerStatus { + progress: Some(format!("{} ({})", self.counter, self.repairs)), + ..Default::default() + } + } + + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + for _i in 0..RC_REPAIR_ITER_COUNT { + let next1 = self + .block_manager + .rc + .rc_table + .range(self.cursor.as_slice()..)? + .next() + .transpose()? + .map(|(k, _)| Hash::try_from(k.as_slice()).unwrap()); + let next2 = self + .block_ref_table + .data + .store + .range(self.cursor.as_slice()..)? + .next() + .transpose()? + .map(|(k, _)| Hash::try_from(&k[..32]).unwrap()); + let next = match (next1, next2) { + (Some(k1), Some(k2)) => std::cmp::min(k1, k2), + (Some(k), None) | (None, Some(k)) => k, + (None, None) => { + info!( + "{}: finished, done {}, fixed {}", + self.name(), + self.counter, + self.repairs + ); + return Ok(WorkerState::Done); + } + }; + + if self.block_manager.rc.recalculate_rc(&next)?.1 { + self.repairs += 1; + } + self.counter += 1; + if let Some(next_incr) = next.increment() { + self.cursor = next_incr; + } else { + info!( + "{}: finished, done {}, fixed {}", + self.name(), + self.counter, + self.repairs + ); + return Ok(WorkerState::Done); + } + } + + Ok(WorkerState::Busy) + } + + async fn wait_for_work(&mut self) -> WorkerState { + unreachable!() + } +} diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs new file mode 100644 index 00000000..17781efe --- /dev/null +++ b/src/garage/secrets.rs @@ -0,0 +1,320 @@ +use std::path::PathBuf; + +use structopt::StructOpt; + +use garage_util::config::Config; +use garage_util::error::Error; + +/// Structure for secret values or paths that are passed as CLI arguments or environment +/// variables, instead of in the config file. +#[derive(StructOpt, Debug, Default, Clone)] +pub struct Secrets { + /// Skip permission check on files containing secrets + #[cfg(unix)] + #[structopt( + long = "allow-world-readable-secrets", + env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS" + )] + pub allow_world_readable_secrets: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml when running the + /// daemon or doing admin operations + #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] + pub rpc_secret: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret + /// when running the daemon or doing admin operations + #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] + pub rpc_secret_file: Option, + + /// Admin API authentication token, replaces admin.admin_token in config.toml when + /// running the Garage daemon + #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] + pub admin_token: Option, + + /// Admin API authentication token file path, replaces admin.admin_token in config.toml + /// and admin-token when running the Garage daemon + #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] + pub admin_token_file: Option, + + /// Metrics API authentication token, replaces admin.metrics_token in config.toml when + /// running the Garage daemon + #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] + pub metrics_token: Option, + + /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml + /// and metrics-token when running the Garage daemon + #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] + pub metrics_token_file: Option, +} + +/// Single function to fill all secrets in the Config struct from their correct source (value +/// from config or CLI param or env variable or read from a file specified in config or CLI +/// param or env variable) +pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result { + let allow_world_readable = secrets + .allow_world_readable_secrets + .unwrap_or(config.allow_world_readable_secrets); + + fill_secret( + &mut config.rpc_secret, + &config.rpc_secret_file, + &secrets.rpc_secret, + &secrets.rpc_secret_file, + "rpc_secret", + allow_world_readable, + )?; + + fill_secret( + &mut config.admin.admin_token, + &config.admin.admin_token_file, + &secrets.admin_token, + &secrets.admin_token_file, + "admin.admin_token", + allow_world_readable, + )?; + fill_secret( + &mut config.admin.metrics_token, + &config.admin.metrics_token_file, + &secrets.metrics_token, + &secrets.metrics_token_file, + "admin.metrics_token", + allow_world_readable, + )?; + + Ok(config) +} + +pub(crate) fn fill_secret( + config_secret: &mut Option, + config_secret_file: &Option, + cli_secret: &Option, + cli_secret_file: &Option, + name: &'static str, + allow_world_readable: bool, +) -> Result<(), Error> { + let cli_value = match (&cli_secret, &cli_secret_file) { + (Some(_), Some(_)) => { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + (Some(secret), None) => Some(secret.to_string()), + (None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?), + (None, None) => None, + }; + + if let Some(val) = cli_value { + if config_secret.is_some() || config_secret_file.is_some() { + debug!("Overriding secret `{}` using value specified using CLI argument or environment variable.", name); + } + + *config_secret = Some(val); + } else if let Some(file_path) = &config_secret_file { + if config_secret.is_some() { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + + *config_secret = Some(read_secret_file(file_path, allow_world_readable)?); + } + + Ok(()) +} + +fn read_secret_file(file_path: &PathBuf, allow_world_readable: bool) -> Result { + if !allow_world_readable { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(file_path)?; + if metadata.mode() & 0o077 != 0 { + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path.display(), metadata.mode()).into()); + } + } + } + + let secret_buf = std::fs::read_to_string(file_path)?; + + // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. + // also editors sometimes add a trailing newline + Ok(String::from(secret_buf.trim_end())) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + + use garage_util::config::read_config; + use garage_util::error::Error; + + use super::*; + + #[test] + fn test_rpc_secret_file_works() -> Result<(), Error> { + let path_secret = mktemp::Temp::new_file()?; + let mut file_secret = File::create(path_secret.as_path())?; + writeln!(file_secret, "foo")?; + drop(file_secret); + + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + let path_secret_path = path_secret.as_path(); + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config); + + // Second configuration file, same as previous one + // except it allows world-readable secrets. + let path_config_allow_world_readable = mktemp::Temp::new_file()?; + let mut file_config_allow_world_readable = + File::create(path_config_allow_world_readable.as_path())?; + writeln!( + file_config_allow_world_readable, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + allow_world_readable_secrets = true + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config_allow_world_readable); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets(config, Secrets::default())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + + // ---- Check non world-readable secrets config ---- + #[cfg(unix)] + { + let secrets_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(true), + ..Default::default() + }; + let secrets_no_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(false), + ..Default::default() + }; + + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(path_secret_path)?; + let mut perm = metadata.permissions(); + perm.set_mode(0o660); + std::fs::set_permissions(path_secret_path, perm)?; + + // Config file that just specifies the path + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_err()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err()); + + // Config file that also specifies to allow world_readable_secrets + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err()); + } + + // ---- Check alternative secrets specified on CLI ---- + + let path_secret2 = mktemp::Temp::new_file()?; + let mut file_secret2 = File::create(path_secret2.as_path())?; + writeln!(file_secret2, "bar")?; + drop(file_secret2); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("baz")); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret_file: Some(path_secret2.clone()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("bar")); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + rpc_secret_file: Some(path_secret2.clone()), + ..Default::default() + } + ) + .is_err()); + + drop(path_secret); + drop(path_secret2); + drop(path_config); + drop(path_config_allow_world_readable); + + Ok(()) + } + + #[test] + fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret= "dummy" + rpc_secret_file = "dummy" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + let config = read_config(path_config.to_path_buf())?; + assert_eq!( + "only one of `rpc_secret` and `rpc_secret_file` can be set", + fill_secrets(config, Secrets::default()) + .unwrap_err() + .to_string() + ); + drop(path_config); + drop(file_config); + Ok(()) + } +} diff --git a/src/garage/server.rs b/src/garage/server.rs index 958089c6..b81ae334 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -6,18 +6,18 @@ use garage_util::background::*; use garage_util::config::*; use garage_util::error::Error; -use garage_api::admin::api_server::AdminApiServer; -use garage_api::s3::api_server::S3ApiServer; +use garage_api_admin::api_server::AdminApiServer; +use garage_api_s3::api_server::S3ApiServer; use garage_model::garage::Garage; use garage_web::WebServer; #[cfg(feature = "k2v")] -use garage_api::k2v::api_server::K2VApiServer; +use garage_api_k2v::api_server::K2VApiServer; use crate::admin::*; +use crate::secrets::{fill_secrets, Secrets}; #[cfg(feature = "telemetry-otlp")] use crate::tracing_setup::*; -use crate::{fill_secrets, Secrets}; async fn wait_from(mut chan: watch::Receiver) { while !*chan.borrow() { @@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver) { pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> { info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; // ---- Initialize Garage internals ---- #[cfg(feature = "metrics")] - let metrics_exporter = opentelemetry_prometheus::exporter().init(); + let metrics_exporter = opentelemetry_prometheus::exporter() + .with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99]) + .with_default_histogram_boundaries(vec![ + 0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1, + 0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60., + 70., 100., + ]) + .init(); info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone())?; @@ -44,7 +51,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone()); info!("Spawning Garage workers..."); - garage.spawn_workers(&background); + garage.spawn_workers(&background)?; if config.admin.trace_sink.is_some() { info!("Initialize tracing..."); @@ -79,9 +86,9 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er "S3 API", tokio::spawn(S3ApiServer::run( garage.clone(), - *s3_bind_addr, + s3_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -94,9 +101,9 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er "K2V API", tokio::spawn(K2VApiServer::run( garage.clone(), - config.k2v_api.as_ref().unwrap().api_bind_addr, + config.k2v_api.as_ref().unwrap().api_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -106,14 +113,10 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er if let Some(web_config) = &config.s3_web { info!("Initializing web server..."); + let web_server = WebServer::new(garage.clone(), &web_config); servers.push(( "Web", - tokio::spawn(WebServer::run( - garage.clone(), - web_config.bind_addr, - web_config.root_domain.clone(), - wait_from(watch_cancel.clone()), - )), + tokio::spawn(web_server.run(web_config.bind_addr.clone(), watch_cancel.clone())), )); } @@ -121,7 +124,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er info!("Launching Admin API server..."); servers.push(( "Admin", - tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))), + tokio::spawn(admin_server.run(admin_bind_addr.clone(), watch_cancel.clone())), )); } @@ -130,20 +133,27 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er warn!("This Garage version is built without the metrics feature"); } - // Stuff runs + if servers.is_empty() { + // Nothing runs except netapp (not in servers) + // Await shutdown signal before proceeding to shutting down netapp + wait_from(watch_cancel).await; + } else { + // Stuff runs - // When a cancel signal is sent, stuff stops + // When a cancel signal is sent, stuff stops - // Collect stuff - for (desc, join_handle) in servers { - if let Err(e) = join_handle.await? { - error!("{} server exited with error: {}", desc, e); - } else { - info!("{} server exited without error.", desc); + // Collect stuff + for (desc, join_handle) in servers { + if let Err(e) = join_handle.await? { + error!("{} server exited with error: {}", desc, e); + } else { + info!("{} server exited without error.", desc); + } } } // Remove RPC handlers for system to break reference cycles + info!("Deregistering RPC handlers for shutdown..."); garage.system.netapp.drop_all_handlers(); opentelemetry::global::shutdown_tracer_provider(); @@ -152,6 +162,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er info!("Netapp exited"); // Drop all references so that stuff can terminate properly + garage.system.cleanup(); drop(garage); // Await for all background tasks to end @@ -172,10 +183,21 @@ fn watch_shutdown_signal() -> watch::Receiver { let mut sigterm = signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler"); let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler"); - tokio::select! { - _ = sigint.recv() => info!("Received SIGINT, shutting down."), - _ = sigterm.recv() => info!("Received SIGTERM, shutting down."), - _ = sighup.recv() => info!("Received SIGHUP, shutting down."), + loop { + tokio::select! { + _ = sigint.recv() => { + info!("Received SIGINT, shutting down."); + break + } + _ = sigterm.recv() => { + info!("Received SIGTERM, shutting down."); + break + } + _ = sighup.recv() => { + info!("Received SIGHUP, reload not supported."); + continue + } + } } send_cancel.send(true).unwrap(); }); diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs index ef4daa5d..7a6612cb 100644 --- a/src/garage/tests/common/client.rs +++ b/src/garage/tests/common/client.rs @@ -1,3 +1,4 @@ +use aws_sdk_s3::config::BehaviorVersion; use aws_sdk_s3::config::Credentials; use aws_sdk_s3::{Client, Config}; @@ -11,6 +12,7 @@ pub fn build_client(key: &Key) -> Client { .endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT)) .region(super::REGION) .credentials_provider(credentials) + .behavior_version(BehaviorVersion::v2024_03_28()) .build(); Client::from_conf(config) diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 4133bb8b..6a8eed38 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -1,15 +1,23 @@ #![allow(dead_code)] use std::collections::HashMap; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use chrono::{offset::Utc, DateTime}; use hmac::{Hmac, Mac}; -use hyper::client::HttpConnector; -use hyper::{Body, Client, Method, Request, Response, Uri}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::header::{ + HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, HOST, +}; +use hyper::{Method, Request, Response, Uri}; +use hyper_util::client::legacy::{connect::HttpConnector, Client}; +use hyper_util::rt::TokioExecutor; use super::garage::{Instance, Key}; -use garage_api::signature; +use garage_api_common::signature; + +pub type Body = FullBody; /// You should ever only use this to send requests AWS sdk won't send, /// like to reproduce behavior of unusual implementations found to be @@ -19,7 +27,7 @@ pub struct CustomRequester { key: Key, uri: Uri, service: &'static str, - client: Client, + client: Client, } impl CustomRequester { @@ -28,7 +36,7 @@ impl CustomRequester { key: key.clone(), uri: instance.s3_uri(), service: "s3", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -37,7 +45,7 @@ impl CustomRequester { key: key.clone(), uri: instance.k2v_uri(), service: "k2v", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -56,6 +64,10 @@ impl CustomRequester { vhost_style: false, } } + + pub fn client(&self) -> &Client { + &self.client + } } pub struct RequestBuilder<'a> { @@ -139,9 +151,9 @@ impl<'a> RequestBuilder<'a> { self } - pub async fn send(&mut self) -> hyper::Result> { + pub async fn send(&mut self) -> Result, String> { // TODO this is a bit incorrect in that path and query params should be url-encoded and - // aren't, but this is good enought for now. + // aren't, but this is good enough for now. let query = query_param_to_string(&self.query_params); let (host, path) = if self.vhost_style { @@ -168,54 +180,123 @@ impl<'a> RequestBuilder<'a> { .unwrap(); let streaming_signer = signer.clone(); - let mut all_headers = self.signed_headers.clone(); + let mut all_headers = self + .signed_headers + .iter() + .map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + }) + .collect::(); let date = now.format(signature::LONG_DATETIME).to_string(); - all_headers.insert("x-amz-date".to_owned(), date); - all_headers.insert("host".to_owned(), host); + all_headers.insert(signature::X_AMZ_DATE, HeaderValue::from_str(&date).unwrap()); + all_headers.insert(HOST, HeaderValue::from_str(&host).unwrap()); - let body_sha = match self.body_signature { + let body_sha = match &self.body_signature { BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(), BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)), - BodySignature::Streaming(size) => { - all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned()); + BodySignature::Streaming { chunk_size } => { all_headers.insert( - "x-amz-decoded-content-length".to_owned(), - self.body.len().to_string(), + CONTENT_ENCODING, + HeaderValue::from_str("aws-chunked").unwrap(), ); - // Get lenght of body by doing the conversion to a streaming body with an + all_headers.insert( + HeaderName::from_static("x-amz-decoded-content-length"), + HeaderValue::from_str(&self.body.len().to_string()).unwrap(), + ); + // Get length of body by doing the conversion to a streaming body with an // invalid signature (we don't know the seed) just to get its length. This - // is a pretty lazy and inefficient way to do it, but it's enought for test + // is a pretty lazy and inefficient way to do it, but it's enough for test // code. all_headers.insert( - "content-length".to_owned(), - to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") - .len() - .to_string(), + CONTENT_LENGTH, + to_streaming_body( + &self.body, + *chunk_size, + String::new(), + signer.clone(), + now, + "", + ) + .len() + .to_string() + .try_into() + .unwrap(), ); "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned() } + BodySignature::StreamingUnsignedTrailer { + chunk_size, + trailer_algorithm, + trailer_value, + } => { + all_headers.insert( + CONTENT_ENCODING, + HeaderValue::from_str("aws-chunked").unwrap(), + ); + all_headers.insert( + HeaderName::from_static("x-amz-decoded-content-length"), + HeaderValue::from_str(&self.body.len().to_string()).unwrap(), + ); + all_headers.insert( + HeaderName::from_static("x-amz-trailer"), + HeaderValue::from_str(&trailer_algorithm).unwrap(), + ); + + all_headers.insert( + CONTENT_LENGTH, + to_streaming_unsigned_trailer_body( + &self.body, + *chunk_size, + &trailer_algorithm, + &trailer_value, + ) + .len() + .to_string() + .try_into() + .unwrap(), + ); + + "STREAMING-UNSIGNED-PAYLOAD-TRAILER".to_owned() + } }; - all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); + all_headers.insert( + signature::X_AMZ_CONTENT_SHA256, + HeaderValue::from_str(&body_sha).unwrap(), + ); - let mut signed_headers = all_headers + let mut signed_headers = all_headers.keys().cloned().collect::>(); + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + let signed_headers_str = signed_headers .iter() - .map(|(k, _)| k.as_ref()) - .collect::>(); - signed_headers.sort(); - let signed_headers = signed_headers.join(";"); + .map(ToString::to_string) + .collect::>() + .join(";"); - all_headers.extend(self.unsigned_headers.clone()); + all_headers.extend(self.unsigned_headers.iter().map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + })); + + let uri = Uri::try_from(&uri).unwrap(); + let query = signature::payload::parse_query_map(&uri).unwrap(); let canonical_request = signature::payload::canonical_request( self.service, &self.method, - &Uri::try_from(&uri).unwrap(), + uri.path(), + &query, &all_headers, &signed_headers, &body_sha, - ); + ) + .unwrap(); let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request); @@ -223,33 +304,72 @@ impl<'a> RequestBuilder<'a> { let signature = hex::encode(signer.finalize().into_bytes()); let authorization = format!( "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}", - self.requester.key.id, scope, signed_headers, signature + self.requester.key.id, scope, signed_headers_str, signature + ); + all_headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&authorization).unwrap(), ); - all_headers.insert("authorization".to_owned(), authorization); let mut request = Request::builder(); - for (k, v) in all_headers { - request = request.header(k, v); - } + *request.headers_mut().unwrap() = all_headers; - let body = if let BodySignature::Streaming(size) = self.body_signature { - to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope) - } else { - self.body.clone() + let body = match &self.body_signature { + BodySignature::Streaming { chunk_size } => to_streaming_body( + &self.body, + *chunk_size, + signature, + streaming_signer, + now, + &scope, + ), + BodySignature::StreamingUnsignedTrailer { + chunk_size, + trailer_algorithm, + trailer_value, + } => to_streaming_unsigned_trailer_body( + &self.body, + *chunk_size, + &trailer_algorithm, + &trailer_value, + ), + _ => self.body.clone(), }; let request = request .uri(uri) .method(self.method.clone()) .body(Body::from(body)) .unwrap(); - self.requester.client.request(request).await + + let result = self + .requester + .client + .request(request) + .await + .map_err(|err| format!("hyper client error: {}", err))?; + + let (head, body) = result.into_parts(); + let body = Body::new( + body.collect() + .await + .map_err(|err| format!("hyper client error in body.collect: {}", err))? + .to_bytes(), + ); + Ok(Response::from_parts(head, body)) } } pub enum BodySignature { Unsigned, Classic, - Streaming(usize), + Streaming { + chunk_size: usize, + }, + StreamingUnsignedTrailer { + chunk_size: usize, + trailer_algorithm: String, + trailer_value: String, + }, } fn query_param_to_string(params: &HashMap>) -> String { @@ -304,3 +424,26 @@ fn to_streaming_body( res } + +fn to_streaming_unsigned_trailer_body( + body: &[u8], + chunk_size: usize, + trailer_algorithm: &str, + trailer_value: &str, +) -> Vec { + let mut res = Vec::with_capacity(body.len()); + for chunk in body.chunks(chunk_size) { + let header = format!("{:x}\r\n", chunk.len()); + res.extend_from_slice(header.as_bytes()); + res.extend_from_slice(chunk); + res.extend_from_slice(b"\r\n"); + } + + res.extend_from_slice(b"0\r\n"); + res.extend_from_slice(trailer_algorithm.as_bytes()); + res.extend_from_slice(b":"); + res.extend_from_slice(trailer_value.as_bytes()); + res.extend_from_slice(b"\n\r\n\r\n"); + + res +} diff --git a/src/garage/tests/common/ext/process.rs b/src/garage/tests/common/ext/process.rs index ba533b6c..8e20bf7c 100644 --- a/src/garage/tests/common/ext/process.rs +++ b/src/garage/tests/common/ext/process.rs @@ -14,42 +14,20 @@ impl CommandExt for process::Command { } fn expect_success_status(&mut self, msg: &str) -> process::ExitStatus { - let status = self.status().expect(msg); - status.expect_success(msg); - status + self.expect_success_output(msg).status } fn expect_success_output(&mut self, msg: &str) -> process::Output { let output = self.output().expect(msg); - output.expect_success(msg); + if !output.status.success() { + panic!( + "{}: command {:?} exited with error {:?}\nSTDOUT: {}\nSTDERR: {}", + msg, + self, + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } output } } - -pub trait OutputExt { - fn expect_success(&self, msg: &str); -} - -impl OutputExt for process::Output { - fn expect_success(&self, msg: &str) { - self.status.expect_success(msg) - } -} - -pub trait ExitStatusExt { - fn expect_success(&self, msg: &str); -} - -impl ExitStatusExt for process::ExitStatus { - fn expect_success(&self, msg: &str) { - if !self.success() { - match self.code() { - Some(code) => panic!( - "Command exited with code {code}: {msg}", - code = code, - msg = msg - ), - None => panic!("Command exited with signal: {msg}", msg = msg), - } - } - } -} diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index d1f0867a..2b0a381c 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -13,7 +13,6 @@ static GARAGE_TEST_SECRET: &str = #[derive(Debug, Default, Clone)] pub struct Key { - pub name: Option, pub id: String, pub secret: String, } @@ -42,6 +41,10 @@ impl Instance { .ok() .unwrap_or_else(|| env::temp_dir().join(format!("garage-integ-test-{}", port))); + let db_engine = env::var("GARAGE_TEST_INTEGRATION_DB_ENGINE") + .ok() + .unwrap_or_else(|| "lmdb".into()); + // Clean test runtime directory if path.exists() { fs::remove_dir_all(&path).expect("Could not clean test runtime directory"); @@ -52,14 +55,16 @@ impl Instance { r#" metadata_dir = "{path}/meta" data_dir = "{path}/data" -db_engine = "lmdb" +db_engine = "{db_engine}" -replication_mode = "1" +replication_factor = 1 rpc_bind_addr = "127.0.0.1:{rpc_port}" rpc_public_addr = "127.0.0.1:{rpc_port}" rpc_secret = "{secret}" +allow_punycode = true + [s3_api] s3_region = "{region}" api_bind_addr = "127.0.0.1:{s3_port}" @@ -96,7 +101,10 @@ api_bind_addr = "127.0.0.1:{admin_port}" .arg("server") .stdout(stdout) .stderr(stderr) - .env("RUST_LOG", "garage=info,garage_api=trace") + .env( + "RUST_LOG", + "garage=debug,garage_api_common=trace,garage_api_s3=trace", + ) .spawn() .expect("Could not start garage"); @@ -209,10 +217,7 @@ api_bind_addr = "127.0.0.1:{admin_port}" assert!(!key.id.is_empty(), "Invalid key: Key ID is empty"); assert!(!key.secret.is_empty(), "Invalid key: Key secret is empty"); - Key { - name: maybe_name.map(String::from), - ..key - } + key } } diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index 71de91bf..39554d4d 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -7,6 +7,7 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -77,10 +78,7 @@ async fn test_batch() { .unwrap() .to_string(), ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, values.get(sk).unwrap().as_bytes()); } diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 20add889..5a347bd9 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -7,6 +7,7 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -83,10 +84,7 @@ async fn test_items_and_indices() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content); // ReadIndex -- now there should be some stuff @@ -152,10 +150,7 @@ async fn test_items_and_indices() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content2); // ReadIndex -- now there should be some stuff @@ -394,10 +389,7 @@ async fn test_item_return_format() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f1: not specified @@ -434,10 +426,7 @@ async fn test_item_return_format() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f3: json diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index 452317c2..7c06cea9 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -1,4 +1,5 @@ use base64::prelude::*; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; use std::time::Duration; @@ -9,6 +10,7 @@ use crate::common; use crate::json_body; #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_item() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-item"); @@ -47,11 +49,8 @@ async fn test_poll_item() { .unwrap() .to_string(); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Initial value"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Initial value"[..]); // Start poll operation let poll = { @@ -95,14 +94,12 @@ async fn test_poll_item() { assert_eq!(poll_res.status(), StatusCode::OK); - let poll_res_body = hyper::body::to_bytes(poll_res.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(poll_res_body, b"New value"); + let poll_res_body = poll_res.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(poll_res_body, b"New value"[..]); } #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_range() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-range"); diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs index 465fc24d..1017330d 100644 --- a/src/garage/tests/k2v/simple.rs +++ b/src/garage/tests/k2v/simple.rs @@ -1,5 +1,6 @@ use crate::common; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -32,9 +33,6 @@ async fn test_simple() { .unwrap(); assert_eq!(res2.status(), StatusCode::OK); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Hello, world!"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Hello, world!"[..]); } diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index ab92bc0a..ef370db3 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -11,15 +11,15 @@ mod k2v; #[cfg(feature = "k2v")] mod k2v_client; -use hyper::{Body, Response}; +use http_body_util::BodyExt; +use hyper::{body::Body, Response}; -pub async fn json_body(res: Response) -> serde_json::Value { - let res_body: serde_json::Value = serde_json::from_slice( - &hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec()[..], - ) - .unwrap(); +pub async fn json_body(res: Response) -> serde_json::Value +where + B: Body, + ::Error: std::fmt::Debug, +{ + let body = res.into_body().collect().await.unwrap().to_bytes(); + let res_body: serde_json::Value = serde_json::from_slice(&body).unwrap(); res_body } diff --git a/src/garage/tests/s3/list.rs b/src/garage/tests/s3/list.rs index bb03f250..1b0c006d 100644 --- a/src/garage/tests/s3/list.rs +++ b/src/garage/tests/s3/list.rs @@ -613,3 +613,63 @@ async fn test_listmultipart() { assert!(r.common_prefixes.is_none()); } } + +#[tokio::test] +async fn test_multichar_delimiter() { + // Test case from dpape from issue #692 with reference results from Amazon + + let ctx = common::context(); + let bucket = ctx.create_bucket("multichardelim"); + + for k in [ + "a/", "a/b/", "a/b/c/", "a/b/c/d", "a/c/", "a/c/b/", "a/c/b/e", + ] { + ctx.client + .put_object() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + // With delimiter / + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert!(r.contents.is_none()); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 1); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/"); + } + + // With delimiter b/ + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("b/") + .send() + .await + .unwrap(); + + let contents = r.contents.unwrap(); + assert_eq!(contents.len(), 2); + assert_eq!(contents[0].key.as_deref().unwrap(), "a/"); + assert_eq!(contents[1].key.as_deref().unwrap(), "a/c/"); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 2); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/b/"); + assert_eq!(common_prefixes[1].prefix.as_deref().unwrap(), "a/c/b/"); + } +} diff --git a/src/garage/tests/s3/mod.rs b/src/garage/tests/s3/mod.rs index 623eb665..e75b1397 100644 --- a/src/garage/tests/s3/mod.rs +++ b/src/garage/tests/s3/mod.rs @@ -1,6 +1,8 @@ mod list; mod multipart; mod objects; +mod presigned; mod simple; +mod ssec; mod streaming_signature; mod website; diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs index 09ae5e5b..cc424f59 100644 --- a/src/garage/tests/s3/multipart.rs +++ b/src/garage/tests/s3/multipart.rs @@ -1,6 +1,7 @@ use crate::common; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::types::{ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart}; +use base64::prelude::*; const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024; @@ -154,7 +155,7 @@ async fn test_multipart_upload() { .await .unwrap(); - assert_eq!(r.content_length, (SZ_5MB * 3) as i64); + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 3) as i64); } { @@ -183,12 +184,159 @@ async fn test_multipart_upload() { .unwrap(); eprintln!("get_object with part_number = {}", part_number); - assert_eq!(o.content_length, SZ_5MB as i64); + assert_eq!(o.content_length.unwrap(), SZ_5MB as i64); assert_bytes_eq!(o.body, data); } } } +#[tokio::test] +async fn test_multipart_with_checksum() { + let ctx = common::context(); + let bucket = ctx.create_bucket("testmpu-cksum"); + + let u1 = vec![0x11; SZ_5MB]; + let u2 = vec![0x22; SZ_5MB]; + let u3 = vec![0x33; SZ_5MB]; + + let ck1 = calculate_sha1(&u1); + let ck2 = calculate_sha1(&u2); + let ck3 = calculate_sha1(&u3); + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .checksum_algorithm(ChecksumAlgorithm::Sha1) + .key("a") + .send() + .await + .unwrap(); + assert!(up.upload_id.is_some()); + + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .checksum_sha1(&ck1) + .body(ByteStream::from(u1.clone())) + .send() + .await + .unwrap(); + + // wrong checksum value should return an error + let err1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .checksum_sha1(&ck1) + .body(ByteStream::from(u2.clone())) + .send() + .await; + assert!(err1.is_err()); + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .checksum_sha1(&ck2) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(3) + .checksum_sha1(&ck3) + .body(ByteStream::from(u3.clone())) + .send() + .await + .unwrap(); + + { + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + let parts = r.parts.unwrap(); + assert_eq!(parts.len(), 3); + assert!(parts[0].checksum_crc32.is_none()); + assert!(parts[0].checksum_crc32_c.is_none()); + assert!(parts[0].checksum_sha256.is_none()); + assert_eq!(parts[0].checksum_sha1.as_deref().unwrap(), ck1); + assert_eq!(parts[1].checksum_sha1.as_deref().unwrap(), ck2); + assert_eq!(parts[2].checksum_sha1.as_deref().unwrap(), ck3); + } + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .checksum_sha1(&ck1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .checksum_sha1(&ck2) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .checksum_sha1(&ck3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .build(); + + let expected_checksum = calculate_sha1( + &vec![ + BASE64_STANDARD.decode(&ck1).unwrap(), + BASE64_STANDARD.decode(&ck2).unwrap(), + BASE64_STANDARD.decode(&ck3).unwrap(), + ] + .concat(), + ); + + let res = ctx + .client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .checksum_sha1(expected_checksum.clone()) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + assert_eq!(res.checksum_sha1, Some(expected_checksum)); +} + #[tokio::test] async fn test_uploadlistpart() { let ctx = common::context(); @@ -249,14 +397,14 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 1); - assert_eq!(ps[0].part_number, 2); + assert_eq!(ps[0].part_number.unwrap(), 2); let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); } let p2 = ctx @@ -286,23 +434,23 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 2); - assert_eq!(ps[0].part_number, 1); + assert_eq!(ps[0].part_number.unwrap(), 1); let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3c484266f9315485694556e6c693bfa2\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); - assert_eq!(ps[1].part_number, 2); + assert_eq!(ps[1].part_number.unwrap(), 2); let sp = &ps[1]; assert!(sp.last_modified.is_some()); assert_eq!( sp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); - assert_eq!(sp.size, SZ_5MB as i64); + assert_eq!(sp.size.unwrap(), SZ_5MB as i64); } { @@ -320,14 +468,14 @@ async fn test_uploadlistpart() { assert!(r.part_number_marker.is_none()); assert_eq!(r.next_part_number_marker.as_deref(), Some("1")); - assert_eq!(r.max_parts, 1_i32); - assert!(r.is_truncated); + assert_eq!(r.max_parts.unwrap(), 1_i32); + assert!(r.is_truncated.unwrap()); assert_eq!(r.key.unwrap(), "a"); assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str()); let parts = r.parts.unwrap(); assert_eq!(parts.len(), 1); let fp = &parts[0]; - assert_eq!(fp.part_number, 1); + assert_eq!(fp.part_number.unwrap(), 1); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3c484266f9315485694556e6c693bfa2\"" @@ -349,19 +497,19 @@ async fn test_uploadlistpart() { r2.part_number_marker.as_ref().unwrap(), r.next_part_number_marker.as_ref().unwrap() ); - assert_eq!(r2.max_parts, 1_i32); + assert_eq!(r2.max_parts.unwrap(), 1_i32); assert_eq!(r2.key.unwrap(), "a"); assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str()); let parts = r2.parts.unwrap(); assert_eq!(parts.len(), 1); let fp = &parts[0]; - assert_eq!(fp.part_number, 2); + assert_eq!(fp.part_number.unwrap(), 2); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); //assert!(r2.is_truncated); // WHY? (this was the test before) - assert!(!r2.is_truncated); + assert!(!r2.is_truncated.unwrap()); } let cmp = CompletedMultipartUpload::builder() @@ -411,7 +559,7 @@ async fn test_uploadlistpart() { .await .unwrap(); - assert_eq!(r.content_length, (SZ_5MB * 2) as i64); + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 2) as i64); } } @@ -624,3 +772,11 @@ async fn test_uploadpartcopy() { assert_eq!(real_obj.len(), exp_obj.len()); assert_eq!(real_obj, exp_obj); } + +fn calculate_sha1(bytes: &[u8]) -> String { + use sha1::{Digest, Sha1}; + + let mut hasher = Sha1::new(); + hasher.update(bytes); + BASE64_STANDARD.encode(&hasher.finalize()[..]) +} diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index 27697d45..53e8231d 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -1,5 +1,6 @@ use crate::common; -use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::primitives::{ByteStream, DateTime}; use aws_sdk_s3::types::{Delete, ObjectIdentifier}; const STD_KEY: &str = "hello world"; @@ -50,9 +51,9 @@ async fn test_putobject() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -86,9 +87,9 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b"hi"); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 2); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 2); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -119,9 +120,156 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b""); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + } +} + +#[tokio::test] +async fn test_precondition() { + let ctx = common::context(); + let bucket = ctx.create_bucket("precondition"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let etag2 = "\"ae4984b984cd984fe98d4efa954dce98\""; + let data = ByteStream::from_static(BODY); + + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let last_modified; + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_match(etag) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + last_modified = o.last_modified.unwrap(); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_match(etag2) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) + ); + } + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_none_match(etag2) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_none_match(etag) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + } + let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0); + let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64()); + let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0); + { + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(newer_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(same_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(older_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + } + { + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(older_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) + ); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(same_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(newer_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); } } @@ -187,6 +335,148 @@ async fn test_getobject() { } } +#[tokio::test] +async fn test_metadata() { + use aws_sdk_s3::primitives::{DateTime, DateTimeFormat}; + + let ctx = common::context(); + let bucket = ctx.create_bucket("testmetadata"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let exp = DateTime::from_secs(10000000000); + let exp2 = DateTime::from_secs(10000500000); + + { + // Note. The AWS client SDK adds a Content-Type header + // with value application/octet-stream if it is not given, + // so here we force it to a known different value. + let data = ByteStream::from_static(BODY); + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .content_type("application/test") + .send() + .await + .unwrap(); + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let o = ctx + .client + .head_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/test"); + assert_eq!(o.cache_control, None); + assert_eq!(o.content_disposition, None); + assert_eq!(o.content_encoding, None); + assert_eq!(o.content_language, None); + assert_eq!(o.expires_string, None); + assert_eq!(o.metadata.unwrap_or_default().len(), 0); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .response_content_type("application/x-dummy-test") + .response_cache_control("ccdummy") + .response_content_disposition("cddummy") + .response_content_encoding("cedummy") + .response_content_language("cldummy") + .response_expires(exp) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test"); + assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); + assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); + } + + { + let data = ByteStream::from_static(BODY); + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .content_type("application/test") + .cache_control("cctest") + .content_disposition("cdtest") + .content_encoding("cetest") + .content_language("cltest") + .expires(exp2) + .metadata("testmeta", "hello people") + .metadata("nice-unicode-meta", "宅配便") + .send() + .await + .unwrap(); + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let o = ctx + .client + .head_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/test"); + assert_eq!(o.cache_control.unwrap().as_str(), "cctest"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cdtest"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cetest"); + assert_eq!(o.content_language.unwrap().as_str(), "cltest"); + assert_eq!( + o.expires_string.unwrap(), + exp2.fmt(DateTimeFormat::HttpDate).unwrap() + ); + let mut meta = o.metadata.unwrap(); + assert_eq!(meta.remove("testmeta").unwrap(), "hello people"); + assert_eq!(meta.remove("nice-unicode-meta").unwrap(), "宅配便"); + assert_eq!(meta.len(), 0); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .response_content_type("application/x-dummy-test") + .response_cache_control("ccdummy") + .response_content_disposition("cddummy") + .response_content_encoding("cedummy") + .response_content_language("cldummy") + .response_expires(exp) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test"); + assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); + assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); + } +} + #[tokio::test] async fn test_deleteobject() { let ctx = common::context(); @@ -205,7 +495,7 @@ async fn test_deleteobject() { .await .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -223,7 +513,7 @@ async fn test_deleteobject() { .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -247,7 +537,7 @@ async fn test_deleteobject() { .client .delete_objects() .bucket(&bucket) - .delete(to_del.build()) + .delete(to_del.build().unwrap()) .send() .await .unwrap(); diff --git a/src/garage/tests/s3/presigned.rs b/src/garage/tests/s3/presigned.rs new file mode 100644 index 00000000..15270361 --- /dev/null +++ b/src/garage/tests/s3/presigned.rs @@ -0,0 +1,72 @@ +use std::time::{Duration, SystemTime}; + +use crate::common; +use aws_sdk_s3::presigning::PresigningConfig; +use bytes::Bytes; +use http_body_util::{BodyExt, Full}; +use hyper::Request; + +const STD_KEY: &str = "hello world"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_presigned_url() { + let ctx = common::context(); + let bucket = ctx.create_bucket("presigned"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let body = Bytes::from(BODY.to_vec()); + + let psc = PresigningConfig::builder() + .start_time(SystemTime::now() - Duration::from_secs(60)) + .expires_in(Duration::from_secs(3600)) + .build() + .unwrap(); + + { + // PutObject + let req = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc.clone()) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("PUT") + .uri(req.uri()) + .body(Full::new(body.clone())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + } + + { + // GetObject + let req = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("GET") + .uri(req.uri()) + .body(Full::new(Bytes::new())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + + let body2 = BodyExt::collect(res.into_body()).await.unwrap().to_bytes(); + assert_eq!(body, body2); + } +} diff --git a/src/garage/tests/s3/ssec.rs b/src/garage/tests/s3/ssec.rs new file mode 100644 index 00000000..d8f11950 --- /dev/null +++ b/src/garage/tests/s3/ssec.rs @@ -0,0 +1,455 @@ +use crate::common::{self, Context}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + +const SSEC_KEY: &str = "u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y="; +const SSEC_KEY_MD5: &str = "jMGbs3GyZkYjJUP6q5jA7g=="; +const SSEC_KEY2: &str = "XkYVk4Z3vVDO2yJaUqCAEZX6lL10voMxtV06d8my/eU="; +const SSEC_KEY2_MD5: &str = "kedo2ab8J1MCjHwJuLTJHw=="; + +const SZ_2MB: usize = 2 * 1024 * 1024; + +#[tokio::test] +async fn test_ssec_object() { + let ctx = common::context(); + let bucket = ctx.create_bucket("sse-c"); + + let bytes1 = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec(); + let bytes2 = (0..400000) + .map(|x| ((x * 3792) % 256) as u8) + .collect::>(); + + for data in vec![bytes1, bytes2] { + let stream = ByteStream::new(data.clone().into()); + + // Write encrypted object + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key("testobj") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .body(stream) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + + test_read_encrypted( + &ctx, + &bucket, + "testobj", + &data, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + + // Test copy from encrypted to non-encrypted + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-dec") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, None); + assert_eq!(r.sse_customer_key_md5, None); + + // Test read decrypted file + let r = ctx + .client + .get_object() + .bucket(&bucket) + .key("test-copy-enc-dec") + .send() + .await + .unwrap(); + assert_bytes_eq!(r.body, &data); + assert_eq!(r.sse_customer_algorithm, None); + assert_eq!(r.sse_customer_key_md5, None); + + // Test copy from non-encrypted to encrypted + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-dec-enc") + .copy_source(format!("{}/test-copy-enc-dec", bucket)) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into())); + + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-dec-enc", + &data, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + + // Test copy from encrypted to encrypted with different keys + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-enc") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into())); + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-enc", + &data, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + + // Test copy from encrypted to encrypted with the same key + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-enc-same") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-enc-same", + &data, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + } +} + +#[tokio::test] +async fn test_multipart_upload() { + let ctx = common::context(); + let bucket = ctx.create_bucket("test-ssec-mpu"); + + let u1 = vec![0x11; SZ_2MB]; + let u2 = vec![0x22; SZ_2MB]; + let u3 = vec![0x33; SZ_2MB]; + let all = [&u1[..], &u2[..], &u3[..]].concat(); + + // Test simple encrypted mpu + { + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("a") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert!(up.upload_id.is_some()); + assert_eq!(up.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(up.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + + let uid = up.upload_id.as_ref().unwrap(); + + let mut etags = vec![]; + for (i, part) in vec![&u1, &u2, &u3].into_iter().enumerate() { + let pu = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number((i + 1) as i32) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .body(ByteStream::from(part.to_vec())) + .send() + .await + .unwrap(); + etags.push(pu.e_tag.unwrap()); + } + + let mut cmp = CompletedMultipartUpload::builder(); + for (i, etag) in etags.into_iter().enumerate() { + cmp = cmp.parts( + CompletedPart::builder() + .part_number((i + 1) as i32) + .e_tag(etag) + .build(), + ); + } + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .multipart_upload(cmp.build()) + .send() + .await + .unwrap(); + + test_read_encrypted( + &ctx, + &bucket, + "a", + &all, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + } + + // Test upload part copy from first object + { + // (setup) Upload a single part object + ctx.client + .put_object() + .bucket(&bucket) + .key("b") + .body(ByteStream::from(u1.clone())) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("target") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(1) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .body(ByteStream::from(u3.clone())) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(2) + .copy_source(format!("{}/a", bucket)) + .copy_source_range("bytes=500-550000") + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(3) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .body(ByteStream::from(u2.clone())) + .send() + .await + .unwrap(); + + let p4 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(4) + .copy_source(format!("{}/b", bucket)) + .copy_source_range("bytes=1500-20500") + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY2) + .copy_source_sse_customer_key_md5(SSEC_KEY2_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(4) + .e_tag(p4.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // (check) Get object + let expected = [&u3[..], &all[500..550001], &u2[..], &u1[1500..20501]].concat(); + test_read_encrypted( + &ctx, + &bucket, + "target", + &expected, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + } +} + +async fn test_read_encrypted( + ctx: &Context, + bucket: &str, + obj_key: &str, + expected_data: &[u8], + enc_key: &str, + enc_key_md5: &str, + wrong_enc_key: &str, + wrong_enc_key_md5: &str, +) { + // Test read encrypted without key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .send() + .await; + assert!( + o.is_err(), + "encrypted file could be read without encryption key" + ); + + // Test read encrypted with wrong key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .sse_customer_key(wrong_enc_key) + .sse_customer_key_md5(wrong_enc_key_md5) + .send() + .await; + assert!( + o.is_err(), + "encrypted file could be read with incorrect encryption key" + ); + + // Test read encrypted with correct key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(enc_key) + .sse_customer_key_md5(enc_key_md5) + .send() + .await + .unwrap(); + assert_bytes_eq!(o.body, expected_data); + assert_eq!(o.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(o.sse_customer_key_md5, Some(enc_key_md5.to_string())); +} diff --git a/src/garage/tests/s3/streaming_signature.rs b/src/garage/tests/s3/streaming_signature.rs index b7a1acae..a86feefc 100644 --- a/src/garage/tests/s3/streaming_signature.rs +++ b/src/garage/tests/s3/streaming_signature.rs @@ -1,5 +1,8 @@ use std::collections::HashMap; +use base64::prelude::*; +use crc32fast::Hasher as Crc32; + use crate::common; use crate::common::ext::CommandExt; use common::custom_requester::BodySignature; @@ -21,18 +24,19 @@ async fn test_putobject_streaming() { let content_type = "text/csv"; let mut headers = HashMap::new(); headers.insert("content-type".to_owned(), content_type.to_owned()); - let _ = ctx + let res = ctx .custom_request .builder(bucket.clone()) .method(Method::PUT) .path(STD_KEY.to_owned()) - .unsigned_headers(headers) + .signed_headers(headers) .vhost_style(true) .body(vec![]) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); // assert_eq!(r.e_tag.unwrap().as_str(), etag); // We return a version ID here @@ -57,15 +61,22 @@ async fn test_putobject_streaming() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { let etag = "\"46cf18a9b447991b450cad3facf5937e\""; - let _ = ctx + let mut crc32 = Crc32::new(); + crc32.update(&BODY[..]); + let crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(crc32.finalize())[..]); + + let mut headers = HashMap::new(); + headers.insert("x-amz-checksum-crc32".to_owned(), crc32.clone()); + + let res = ctx .custom_request .builder(bucket.clone()) .method(Method::PUT) @@ -73,11 +84,13 @@ async fn test_putobject_streaming() { //fail .path("abc".to_owned()) .vhost_style(true) + .signed_headers(headers) .body(BODY.to_vec()) - .body_signature(BodySignature::Streaming(16)) + .body_signature(BodySignature::Streaming { chunk_size: 16 }) .send() .await .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); // assert_eq!(r.e_tag.unwrap().as_str(), etag); // assert!(r.version_id.is_some()); @@ -88,6 +101,7 @@ async fn test_putobject_streaming() { .bucket(&bucket) //.key(CTRL_KEY) .key("abc") + .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) .send() .await .unwrap(); @@ -95,9 +109,145 @@ async fn test_putobject_streaming() { assert_bytes_eq!(o.body, BODY); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 62); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 62); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + assert_eq!(o.checksum_crc32.unwrap(), crc32); + } +} + +#[tokio::test] +async fn test_putobject_streaming_unsigned_trailer() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putobject-streaming-unsigned-trailer"); + + { + // Send an empty object (can serve as a directory marker) + // with a content type + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let mut headers = HashMap::new(); + headers.insert("content-type".to_owned(), content_type.to_owned()); + + let empty_crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(Crc32::new().finalize())[..]); + + let res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + .path(STD_KEY.to_owned()) + .signed_headers(headers) + .vhost_style(true) + .body(vec![]) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 10, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: empty_crc32, + }) + .send() + .await + .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // We return a version ID here + // We should check if Amazon is returning one when versioning is not enabled + // assert!(r.version_id.is_some()); + + //let _version = r.version_id.unwrap(); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + // We do not return version ID + // We should check if Amazon is returning one when versioning is not enabled + // assert_eq!(o.version_id.unwrap(), _version); + assert_eq!(o.content_type.unwrap(), content_type); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + } + + { + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + + let mut crc32 = Crc32::new(); + crc32.update(&BODY[..]); + let crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(crc32.finalize())[..]); + + // try sending with wrong crc32, check that it fails + let err_res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 16, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: "2Yp9Yw==".into(), + }) + .send() + .await + .unwrap(); + assert!( + err_res.status().is_client_error(), + "got response: {:?}", + err_res + ); + + let res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 16, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: crc32.clone(), + }) + .send() + .await + .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + //.key(CTRL_KEY) + .key("abc") + .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, BODY); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length.unwrap(), 62); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + assert_eq!(o.checksum_crc32.unwrap(), crc32); } } @@ -119,7 +269,7 @@ async fn test_create_bucket_streaming() { .custom_request .builder(bucket.to_owned()) .method(Method::PUT) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); @@ -174,7 +324,7 @@ async fn test_put_website_streaming() { .method(Method::PUT) .query_params(query) .body(website_config.as_bytes().to_vec()) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); @@ -187,7 +337,7 @@ async fn test_put_website_streaming() { .await .unwrap(); - assert_eq!(o.index_document.unwrap().suffix.unwrap(), "home.html"); - assert_eq!(o.error_document.unwrap().key.unwrap(), "err/error.html"); + assert_eq!(o.index_document.unwrap().suffix, "home.html"); + assert_eq!(o.error_document.unwrap().key, "err/error.html"); } } diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index eeafb5fa..bbac3de5 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -8,15 +8,19 @@ use aws_sdk_s3::{ types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, }; use http::{Request, StatusCode}; -use hyper::{ - body::{to_bytes, Body}, - Client, -}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::body::Bytes; +use hyper::header::LOCATION; +use hyper_util::client::legacy::Client; +use hyper_util::rt::TokioExecutor; use serde_json::json; const BODY: &[u8; 16] = b"

bonjour

"; const BODY_ERR: &[u8; 6] = b"erreur"; +pub type Body = FullBody; + #[tokio::test] async fn test_website() { const BCKT_NAME: &str = "my-website"; @@ -34,14 +38,14 @@ async fn test_website() { .await .unwrap(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -49,7 +53,7 @@ async fn test_website() { assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BodyExt::collect(resp.into_body()).await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -58,10 +62,9 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -87,7 +90,7 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); @@ -103,14 +106,14 @@ async fn test_website() { "http://127.0.0.1:{0}/check?domain={1}", ctx.garage.admin_port, bname )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; - let mut admin_resp = client.request(admin_req()).await.unwrap(); + let admin_resp = client.request(admin_req()).await.unwrap(); assert_eq!(admin_resp.status(), StatusCode::OK); assert_eq!( - to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), + admin_resp.into_body().collect().await.unwrap().to_bytes(), format!("Domain '{bname}' is managed by Garage").as_bytes() ); } @@ -124,7 +127,7 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -133,10 +136,9 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -181,8 +183,18 @@ async fn test_website_s3_api() { .unwrap(); let conf = WebsiteConfiguration::builder() - .index_document(IndexDocument::builder().suffix("home.html").build()) - .error_document(ErrorDocument::builder().key("err/error.html").build()) + .index_document( + IndexDocument::builder() + .suffix("home.html") + .build() + .unwrap(), + ) + .error_document( + ErrorDocument::builder() + .key("err/error.html") + .build() + .unwrap(), + ) .build(); ctx.client @@ -201,9 +213,11 @@ async fn test_website_s3_api() { .allowed_methods("GET") .allowed_methods("PUT") .allowed_origins("*") - .build(), + .build() + .unwrap(), ) - .build(); + .build() + .unwrap(); ctx.client .put_bucket_cors() @@ -222,24 +236,21 @@ async fn test_website_s3_api() { .await .unwrap(); - let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap(); + let main_rule = cors_res.cors_rules().iter().next().unwrap(); assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule"); assert_eq!( main_rule.allowed_headers.as_ref().unwrap(), &vec!["*".to_string()] ); + assert_eq!(&main_rule.allowed_origins, &vec!["*".to_string()]); assert_eq!( - main_rule.allowed_origins.as_ref().unwrap(), - &vec!["*".to_string()] - ); - assert_eq!( - main_rule.allowed_methods.as_ref().unwrap(), + &main_rule.allowed_methods, &vec!["GET".to_string(), "PUT".to_string()] ); } - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); // Test direct requests with CORS { @@ -248,10 +259,10 @@ async fn test_website_s3_api() { .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -259,7 +270,7 @@ async fn test_website_s3_api() { "*" ); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -273,18 +284,45 @@ async fn test_website_s3_api() { ctx.garage.web_port )) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY_ERR.as_ref() ); } + // Test x-amz-website-redirect-location + { + ctx.client + .put_object() + .bucket(&bucket) + .key("test-redirect.html") + .website_redirect_location("https://perdu.com") + .send() + .await + .unwrap(); + + let req = Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/test-redirect.html", + ctx.garage.web_port + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .body(Body::new(Bytes::new())) + .unwrap(); + + let resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), StatusCode::MOVED_PERMANENTLY); + assert_eq!(resp.headers().get(LOCATION).unwrap(), "https://perdu.com"); + } + // Test CORS with an allowed preflight request { let req = Request::builder() @@ -293,10 +331,10 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -304,7 +342,7 @@ async fn test_website_s3_api() { "*" ); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -317,14 +355,14 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "DELETE") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -358,14 +396,14 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -384,20 +422,15 @@ async fn test_website_s3_api() { .method("GET") .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY_ERR.as_ref() - ); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); + let resp_bytes = resp.into_body().collect().await.unwrap().to_bytes(); + assert_ne!(resp_bytes, BODY_ERR.as_ref()); + assert_ne!(resp_bytes, BODY.as_ref()); } } @@ -405,13 +438,13 @@ async fn test_website_s3_api() { async fn test_website_check_domain() { let ctx = common::context(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let admin_req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/check", ctx.garage.admin_port)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -435,7 +468,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -459,7 +492,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=foobar", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -483,7 +516,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=%E2%98%B9", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -500,3 +533,118 @@ async fn test_website_check_domain() { }) ); } + +#[tokio::test] +async fn test_website_puny() { + const BCKT_NAME: &str = "xn--pda.eu"; + let ctx = common::context(); + let bucket = ctx.create_bucket(BCKT_NAME); + + let data = ByteStream::from_static(BODY); + + ctx.client + .put_object() + .bucket(&bucket) + .key("index.html") + .body(data) + .send() + .await + .unwrap(); + + let client = Client::builder(TokioExecutor::new()).build_http(); + + let req = |suffix| { + Request::builder() + .method("GET") + .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) + .header("Host", format!("{}{}", BCKT_NAME, suffix)) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + let mut resp = client.request(req("")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.into_body().collect().await.unwrap().to_bytes(), + BODY.as_ref() + ); + + resp = client.request(req(".web.garage")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.into_body().collect().await.unwrap().to_bytes(), + BODY.as_ref() + ); + + for bname in [ + BCKT_NAME.to_string(), + format!("{BCKT_NAME}.web.garage"), + format!("{BCKT_NAME}.s3.garage"), + ] { + let admin_req = || { + Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{0}/check?domain={1}", + ctx.garage.admin_port, bname + )) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + let admin_resp = client.request(admin_req()).await.unwrap(); + assert_eq!(admin_resp.status(), StatusCode::OK); + assert_eq!( + admin_resp.into_body().collect().await.unwrap().to_bytes(), + format!("Domain '{bname}' is managed by Garage").as_bytes() + ); + } +} + +#[tokio::test] +async fn test_website_object_not_found() { + const BCKT_NAME: &str = "not-found"; + let ctx = common::context(); + let _bucket = ctx.create_bucket(BCKT_NAME); + + let client = Client::builder(TokioExecutor::new()).build_http(); + + let req = |suffix| { + Request::builder() + .method("GET") + .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) + .header("Host", format!("{}{}", BCKT_NAME, suffix)) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + let resp = client.request(req("")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // the error we return by default are *not* xml + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html; charset=utf-8" + ); + let result = String::from_utf8( + resp.into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(), + ) + .unwrap(); + assert!(result.contains("not found")); +} diff --git a/src/k2v-client/Cargo.toml b/src/k2v-client/Cargo.toml index 2ccb9fe5..bbd09b19 100644 --- a/src/k2v-client/Cargo.toml +++ b/src/k2v-client/Cargo.toml @@ -9,29 +9,31 @@ repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" [dependencies] -base64 = "0.21" -sha2 = "0.10" -hex = "0.4" -http = "0.2" -log = "0.4" -aws-sigv4 = "0.55" -percent-encoding = "2.2" -hyper = { version = "0.14", default-features = false, features = ["client", "http1", "http2"] } -hyper-rustls = { version = "0.24", features = ["http2"] } -serde = { version = "1.0", features = [ "derive" ] } -serde_json = "1.0" -thiserror = "1.0" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +base64.workspace = true +sha2.workspace = true +hex.workspace = true +http.workspace = true +http-body-util.workspace = true +log.workspace = true +aws-sigv4.workspace = true +aws-sdk-config.workspace = true +percent-encoding.workspace = true +hyper = { workspace = true, default-features = false, features = ["http1", "http2"] } +hyper-util.workspace = true +hyper-rustls.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true # cli deps -clap = { version = "4.1", optional = true, features = ["derive", "env"] } +clap = { workspace = true, optional = true } format_table = { workspace = true, optional = true } -tracing = { version = "0.1", optional = true } -tracing-subscriber = { version = "0.3", optional = true, features = ["env-filter"] } +tracing-subscriber = { workspace = true, optional = true } [features] -cli = ["clap", "tokio/fs", "tokio/io-std", "tracing", "tracing-subscriber", "format_table"] +cli = ["clap", "tokio/fs", "tokio/io-std", "tracing-subscriber", "format_table"] [lib] path = "lib.rs" diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs index b9461c89..b1c2169b 100644 --- a/src/k2v-client/bin/k2v-cli.rs +++ b/src/k2v-client/bin/k2v-cli.rs @@ -54,7 +54,7 @@ enum Command { partition_key: String, /// Sort key to read from sort_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: ReadOutputKind, }, @@ -70,7 +70,7 @@ enum Command { /// Timeout, in seconds #[clap(short = 'T', long)] timeout: Option, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: ReadOutputKind, }, @@ -87,7 +87,7 @@ enum Command { /// Timeout, in seconds #[clap(short = 'T', long)] timeout: Option, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, }, @@ -103,7 +103,7 @@ enum Command { }, /// List partition keys ReadIndex { - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Output only partition keys matching this filter @@ -114,7 +114,7 @@ enum Command { ReadRange { /// Partition key to read from partition_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Output only sort keys matching this filter @@ -125,7 +125,7 @@ enum Command { DeleteRange { /// Partition key to delete from partition_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Delete only sort keys matching this filter @@ -185,10 +185,10 @@ struct ReadOutputKind { /// Raw output. Conflicts generate error, causality token is not returned #[clap(short, long, group = "output-kind")] raw: bool, - /// Human formated output + /// Human formatted output #[clap(short = 'H', long, group = "output-kind")] human: bool, - /// JSON formated output + /// JSON formatted output #[clap(short, long, group = "output-kind")] json: bool, } @@ -207,7 +207,7 @@ impl ReadOutputKind { let mut val = val.value; if val.len() != 1 { eprintln!( - "Raw mode can only read non-concurent values, found {} values, expected 1", + "Raw mode can only read non-concurrent values, found {} values, expected 1", val.len() ); exit(1); @@ -265,10 +265,10 @@ impl ReadOutputKind { #[derive(Parser, Debug)] #[clap(group = clap::ArgGroup::new("output-kind").multiple(false).required(false))] struct BatchOutputKind { - /// Human formated output + /// Human formatted output #[clap(short = 'H', long, group = "output-kind")] human: bool, - /// JSON formated output + /// JSON formatted output #[clap(short, long, group = "output-kind")] json: bool, } diff --git a/src/k2v-client/error.rs b/src/k2v-client/error.rs index 564ce497..96f5674a 100644 --- a/src/k2v-client/error.rs +++ b/src/k2v-client/error.rs @@ -22,12 +22,14 @@ pub enum Error { Http(#[from] http::Error), #[error("hyper error: {0}")] Hyper(#[from] hyper::Error), + #[error("hyper client error: {0}")] + HyperClient(#[from] hyper_util::client::legacy::Error), #[error("invalid header: {0}")] Header(#[from] hyper::header::ToStrError), #[error("deserialization error: {0}")] Deserialization(#[from] serde_json::Error), #[error("invalid signature parameters: {0}")] - SignParameters(#[from] aws_sigv4::signing_params::BuildError), + SignParameters(#[from] aws_sigv4::sign::v4::signing_params::BuildError), #[error("could not sign request: {0}")] SignRequest(#[from] aws_sigv4::http_request::SigningError), #[error("request timed out")] diff --git a/src/k2v-client/lib.rs b/src/k2v-client/lib.rs index 4aa7a20a..fe8fd3e0 100644 --- a/src/k2v-client/lib.rs +++ b/src/k2v-client/lib.rs @@ -9,11 +9,15 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC}; use http::header::{ACCEPT, CONTENT_TYPE}; use http::status::StatusCode; use http::{HeaderName, HeaderValue, Request}; -use hyper::{body::Bytes, Body}; -use hyper::{client::connect::HttpConnector, Client as HttpClient}; +use http_body_util::{BodyExt, Full as FullBody}; +use hyper::body::Bytes; use hyper_rustls::HttpsConnector; +use hyper_util::client::legacy::{connect::HttpConnector, Client as HttpClient}; +use hyper_util::rt::TokioExecutor; -use aws_sigv4::http_request::{sign, SignableRequest, SigningParams, SigningSettings}; +use aws_sdk_config::config::Credentials; +use aws_sigv4::http_request::{sign, SignableBody, SignableRequest, SigningSettings}; +use aws_sigv4::sign::v4::SigningParams; use serde::de::Error as DeError; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -22,6 +26,8 @@ mod error; pub use error::Error; +pub type Body = FullBody; + const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_POLL_TIMEOUT: Duration = Duration::from_secs(300); const SERVICE: &str = "k2v"; @@ -53,19 +59,29 @@ pub struct K2vClientConfig { pub struct K2vClient { config: K2vClientConfig, user_agent: HeaderValue, - client: HttpClient>, + client: HttpClient, Body>, } impl K2vClient { /// Create a new K2V client. pub fn new(config: K2vClientConfig) -> Result { let connector = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() + .with_native_roots()? .https_or_http() .enable_http1() .enable_http2() .build(); - let client = HttpClient::builder().build(connector); + let client = HttpClient::builder(TokioExecutor::new()).build(connector); + Self::new_with_client(config, client) + } + + /// Create a new K2V client with an external client. + /// Useful for example if you plan on creating many clients but you want to mutualize the + /// underlying thread pools & co. + pub fn new_with_client( + config: K2vClientConfig, + client: HttpClient, Body>, + ) -> Result { let user_agent: std::borrow::Cow = match &config.user_agent { Some(ua) => ua.into(), None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(), @@ -330,7 +346,7 @@ impl K2vClient { .collect()) } - /// Perform a DeleteBatch request, deleting mutiple values or range of values at once, without + /// Perform a DeleteBatch request, deleting multiple values or range of values at once, without /// providing causality information. pub async fn delete_batch(&self, operations: &[BatchDeleteOp<'_>]) -> Result, Error> { let url = self.build_url(None, &[("delete", "")]); @@ -363,21 +379,37 @@ impl K2vClient { // Sign request let signing_settings = SigningSettings::default(); + let identity = Credentials::new( + &self.config.aws_access_key_id, + &self.config.aws_secret_access_key, + None, + None, + "k2v-client", + ) + .into(); let signing_params = SigningParams::builder() - .access_key(&self.config.aws_access_key_id) - .secret_key(&self.config.aws_secret_access_key) + .identity(&identity) .region(&self.config.region) - .service_name(SERVICE) + .name(SERVICE) .time(SystemTime::now()) .settings(signing_settings) - .build()?; + .build()? + .into(); // Convert the HTTP request into a signable request - let signable_request = SignableRequest::from(&req); + let signable_request = SignableRequest::new( + req.method().as_str(), + req.uri().to_string(), + // TODO: get rid of Unwrap + req.headers() + .iter() + .map(|(x, y)| (x.as_str(), y.to_str().unwrap())), + SignableBody::Bytes(req.body().as_ref()), + )?; // Sign and then apply the signature to the request let (signing_instructions, _signature) = sign(signable_request, &signing_params)?.into_parts(); - signing_instructions.apply_to_request(&mut req); + signing_instructions.apply_to_request_http1x(&mut req); // Send and wait for timeout let res = tokio::select! { @@ -398,12 +430,16 @@ impl K2vClient { }; let body = match res.status { - StatusCode::OK => hyper::body::to_bytes(body).await?, + StatusCode::OK => BodyExt::collect(body).await?.to_bytes(), StatusCode::NO_CONTENT => Bytes::new(), StatusCode::NOT_FOUND => return Err(Error::NotFound), StatusCode::NOT_MODIFIED => Bytes::new(), s => { - let err_body = hyper::body::to_bytes(body).await.unwrap_or_default(); + let err_body = body + .collect() + .await + .map(|x| x.to_bytes()) + .unwrap_or_default(); let err_body_str = std::str::from_utf8(&err_body) .map(String::from) .unwrap_or_else(|_| BASE64_STANDARD.encode(&err_body)); @@ -451,7 +487,11 @@ impl K2vClient { } fn build_url>(&self, partition_key: Option<&str>, query: &[(&str, V)]) -> String { - let mut url = format!("{}/{}", self.config.endpoint, self.config.bucket); + let mut url = format!( + "{}/{}", + self.config.endpoint.trim_end_matches('/'), + self.config.bucket + ); if let Some(pk) = partition_key { url.push('/'); url.extend(utf8_percent_encode(pk, &PATH_ENCODE_SET)); diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index bb3e2b11..289c0024 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -19,31 +19,29 @@ garage_rpc.workspace = true garage_table.workspace = true garage_block.workspace = true garage_util.workspace = true +garage_net.workspace = true -async-trait = "0.1.7" -arc-swap = "1.0" -blake2 = "0.10" -chrono = "0.4" -err-derive = "0.3" -hex = "0.4" -base64 = "0.21" -tracing = "0.1" -rand = "0.8" -zstd = { version = "0.12", default-features = false } +async-trait.workspace = true +blake2.workspace = true +chrono.workspace = true +thiserror.workspace = true +hex.workspace = true +http.workspace = true +base64.workspace = true +parse_duration.workspace = true +tracing.workspace = true +rand.workspace = true +zstd.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true +serde_bytes.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -opentelemetry = "0.17" - -netapp = "0.5" +futures.workspace = true +tokio.workspace = true [features] -default = [ "sled", "lmdb", "sqlite" ] +default = [ "lmdb", "sqlite" ] k2v = [ "garage_util/k2v" ] lmdb = [ "garage_db/lmdb" ] -sled = [ "garage_db/sled" ] sqlite = [ "garage_db/sqlite" ] +fjall = [ "garage_db/fjall" ] diff --git a/src/model/bucket_alias_table.rs b/src/model/bucket_alias_table.rs index 54d7fbad..276d0d1c 100644 --- a/src/model/bucket_alias_table.rs +++ b/src/model/bucket_alias_table.rs @@ -22,14 +22,10 @@ mod v08 { pub use v08::*; impl BucketAlias { - pub fn new(name: String, ts: u64, bucket_id: Option) -> Option { - if !is_valid_bucket_name(&name) { - None - } else { - Some(BucketAlias { - name, - state: crdt::Lww::raw(ts, bucket_id), - }) + pub fn new(name: String, ts: u64, bucket_id: Option) -> Self { + BucketAlias { + name, + state: crdt::Lww::raw(ts, bucket_id), } } @@ -80,7 +76,7 @@ impl TableSchema for BucketAliasTable { /// In the case of Garage, bucket names must not be hex-encoded /// 32 byte string, which is excluded thanks to the /// maximum length of 63 bytes given in the spec. -pub fn is_valid_bucket_name(n: &str) -> bool { +pub fn is_valid_bucket_name(n: &str, puny: bool) -> bool { // Bucket names must be between 3 and 63 characters n.len() >= 3 && n.len() <= 63 // Bucket names must be composed of lowercase letters, numbers, @@ -89,10 +85,12 @@ pub fn is_valid_bucket_name(n: &str) -> bool { // Bucket names must start and end with a letter or a number && !n.starts_with(&['-', '.'][..]) && !n.ends_with(&['-', '.'][..]) - // Bucket names must not be formated as an IP address + // Bucket names must not be formatted as an IP address && n.parse::().is_err() - // Bucket names must not start wih "xn--" - && !n.starts_with("xn--") + // Bucket names must not start with "xn--" + && (!n.starts_with("xn--") || puny) + // We are a bit stricter, to properly restrict punycode in all labels + && (!n.contains(".xn--") || puny) // Bucket names must not end with "-s3alias" && !n.ends_with("-s3alias") } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 4c48a76f..f1cc032e 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -14,7 +14,7 @@ mod v08 { /// A bucket is a collection of objects /// /// Its parameters are not directly accessible as: - /// - It must be possible to merge paramaters, hence the use of a LWW CRDT. + /// - It must be possible to merge parameters, hence the use of a LWW CRDT. /// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Bucket { @@ -126,7 +126,7 @@ impl AutoCrdt for BucketQuotas { } impl BucketParams { - /// Create an empty BucketParams with no authorized keys and no website accesss + /// Create an empty BucketParams with no authorized keys and no website access fn new() -> Self { BucketParams { creation_date: now_msec(), @@ -191,6 +191,13 @@ impl Bucket { } } + pub fn present(id: Uuid, params: BucketParams) -> Self { + Bucket { + id, + state: crdt::Deletable::present(params), + } + } + /// Returns true if this represents a deleted bucket pub fn is_deleted(&self) -> bool { self.state.is_deleted() diff --git a/src/model/garage.rs b/src/model/garage.rs index 8c9a3af3..f4f6f693 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -1,6 +1,7 @@ +use std::str::FromStr; use std::sync::Arc; -use netapp::NetworkKey; +use garage_net::NetworkKey; use garage_db as db; @@ -9,7 +10,7 @@ use garage_util::config::*; use garage_util::error::*; use garage_util::persister::PersisterShared; -use garage_rpc::replication_mode::ReplicationMode; +use garage_rpc::replication_mode::*; use garage_rpc::system::System; use garage_block::manager::*; @@ -39,8 +40,8 @@ pub struct Garage { /// The set of background variables that can be viewed/modified at runtime pub bg_vars: vars::BgVars, - /// The replication mode of this cluster - pub replication_mode: ReplicationMode, + /// The replication factor of this cluster + pub replication_factor: ReplicationFactor, /// The local database pub db: db::Db, @@ -56,6 +57,9 @@ pub struct Garage { /// Table containing api keys pub key_table: Arc>, + /// Lock to prevent concurrent modification of buckets and access keys + bucket_lock: tokio::sync::Mutex<()>, + /// Table containing S3 objects pub object_table: Arc>, /// Counting table containing object counters @@ -110,152 +114,59 @@ impl Garage { } info!("Opening database..."); - let mut db_path = config.metadata_dir.clone(); - let db = match config.db_engine.as_str() { - // ---- Sled DB ---- - #[cfg(feature = "sled")] - "sled" => { - if config.metadata_fsync { - return Err(Error::Message(format!( - "`metadata_fsync = true` is not supported with the Sled database engine" - ))); - } - db_path.push("db"); - info!("Opening Sled database at: {}", db_path.display()); - let db = db::sled_adapter::sled::Config::default() - .path(&db_path) - .cache_capacity(config.sled_cache_capacity as u64) - .flush_every_ms(Some(config.sled_flush_every_ms)) - .open() - .ok_or_message("Unable to open sled DB")?; - db::sled_adapter::SledDb::init(db) - } - #[cfg(not(feature = "sled"))] - "sled" => return Err(Error::Message("sled db not available in this build".into())), - // ---- Sqlite DB ---- - #[cfg(feature = "sqlite")] - "sqlite" | "sqlite3" | "rusqlite" => { - db_path.push("db.sqlite"); - info!("Opening Sqlite database at: {}", db_path.display()); - let db = db::sqlite_adapter::rusqlite::Connection::open(db_path) - .and_then(|db| { - db.pragma_update(None, "journal_mode", &"WAL")?; - if config.metadata_fsync { - db.pragma_update(None, "synchronous", &"NORMAL")?; - } else { - db.pragma_update(None, "synchronous", &"OFF")?; - } - Ok(db) - }) - .ok_or_message("Unable to open sqlite DB")?; - db::sqlite_adapter::SqliteDb::init(db) - } - #[cfg(not(feature = "sqlite"))] - "sqlite" | "sqlite3" | "rusqlite" => { - return Err(Error::Message( - "sqlite db not available in this build".into(), - )) - } - // ---- LMDB DB ---- - #[cfg(feature = "lmdb")] - "lmdb" | "heed" => { - db_path.push("db.lmdb"); - info!("Opening LMDB database at: {}", db_path.display()); - std::fs::create_dir_all(&db_path) - .ok_or_message("Unable to create LMDB data directory")?; - let map_size = match config.lmdb_map_size { - v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(), - v => v - (v % 4096), - }; - - use db::lmdb_adapter::heed; - let mut env_builder = heed::EnvOpenOptions::new(); - env_builder.max_dbs(100); - env_builder.max_readers(500); - env_builder.map_size(map_size); - unsafe { - env_builder.flag(heed::flags::Flags::MdbNoMetaSync); - if !config.metadata_fsync { - env_builder.flag(heed::flags::Flags::MdbNoSync); - } - } - let db = match env_builder.open(&db_path) { - Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => { - return Err(Error::Message( - "OutOfMemory error while trying to open LMDB database. This can happen \ - if your operating system is not allowing you to use sufficient virtual \ - memory address space. Please check that no limit is set (ulimit -v). \ - You may also try to set a smaller `lmdb_map_size` configuration parameter. \ - On 32-bit machines, you should probably switch to another database engine.".into())) - } - x => x.ok_or_message("Unable to open LMDB DB")?, - }; - db::lmdb_adapter::LmdbDb::init(db) - } - #[cfg(not(feature = "lmdb"))] - "lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())), - // ---- Unavailable DB engine ---- - e => { - return Err(Error::Message(format!( - "Unsupported DB engine: {} (options: {})", - e, - vec![ - #[cfg(feature = "sled")] - "sled", - #[cfg(feature = "sqlite")] - "sqlite", - #[cfg(feature = "lmdb")] - "lmdb", - ] - .join(", ") - ))); - } + let db_engine = db::Engine::from_str(&config.db_engine) + .ok_or_message("Invalid `db_engine` value in configuration file")?; + let db_path = db_engine.db_path(&config.metadata_dir); + let db_opt = db::OpenOpt { + fsync: config.metadata_fsync, + lmdb_map_size: match config.lmdb_map_size { + v if v == usize::default() => None, + v => Some(v), + }, + fjall_block_cache_size: match config.fjall_block_cache_size { + v if v == usize::default() => None, + v => Some(v), + }, }; + let db = db::open_db(&db_path, db_engine, &db_opt) + .ok_or_message("Unable to open metadata db")?; + info!("Initializing RPC..."); let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message( "rpc_secret value is missing, not present in config file or in environment", )?) .ok() .and_then(|x| NetworkKey::from_slice(&x)) - .ok_or_message("Invalid RPC secret key")?; + .ok_or_message("Invalid RPC secret key: expected 32 bytes of random hex, please check the documentation for requirements")?; - let replication_mode = ReplicationMode::parse(&config.replication_mode) - .ok_or_message("Invalid replication_mode in config file.")?; + let (replication_factor, consistency_mode) = parse_replication_mode(&config)?; info!("Initialize background variable system..."); let mut bg_vars = vars::BgVars::new(); info!("Initialize membership management system..."); - let system = System::new(network_key, replication_mode, &config)?; + let system = System::new(network_key, replication_factor, consistency_mode, &config)?; let data_rep_param = TableShardedReplication { system: system.clone(), - replication_factor: replication_mode.replication_factor(), - write_quorum: replication_mode.write_quorum(), + replication_factor: replication_factor.into(), + write_quorum: replication_factor.write_quorum(consistency_mode), read_quorum: 1, }; let meta_rep_param = TableShardedReplication { system: system.clone(), - replication_factor: replication_mode.replication_factor(), - write_quorum: replication_mode.write_quorum(), - read_quorum: replication_mode.read_quorum(), + replication_factor: replication_factor.into(), + write_quorum: replication_factor.write_quorum(consistency_mode), + read_quorum: replication_factor.read_quorum(consistency_mode), }; let control_rep_param = TableFullReplication { system: system.clone(), - max_faults: replication_mode.control_write_max_faults(), }; info!("Initialize block manager..."); - let block_manager = BlockManager::new( - &db, - config.data_dir.clone(), - config.data_fsync, - config.compression_level, - data_rep_param, - system.clone(), - )?; + let block_manager = BlockManager::new(&db, &config, data_rep_param, system.clone())?; block_manager.register_bg_vars(&mut bg_vars); // ---- admin tables ---- @@ -332,17 +243,26 @@ impl Garage { #[cfg(feature = "k2v")] let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param); + // ---- setup block refcount recalculation ---- + // this function can be used to fix inconsistencies in the RC table + block_manager.set_recalc_rc(vec![ + block_ref_recount_fn(&block_ref_table), + // other functions could be added here if we had other tables + // that hold references to data blocks + ]); + // -- done -- Ok(Arc::new(Self { config, bg_vars, - replication_mode, + replication_factor, db, system, block_manager, bucket_table, bucket_alias_table, key_table, + bucket_lock: tokio::sync::Mutex::new(()), object_table, object_counter_table, mpu_table, @@ -355,7 +275,7 @@ impl Garage { })) } - pub fn spawn_workers(self: &Arc, bg: &BackgroundRunner) { + pub fn spawn_workers(self: &Arc, bg: &BackgroundRunner) -> Result<(), Error> { self.block_manager.spawn_workers(bg); self.bucket_table.spawn_workers(bg); @@ -376,15 +296,37 @@ impl Garage { #[cfg(feature = "k2v")] self.k2v.spawn_workers(bg); + + if let Some(itv) = self.config.metadata_auto_snapshot_interval.as_deref() { + let interval = parse_duration::parse(itv) + .ok_or_message("Invalid `metadata_auto_snapshot_interval`")?; + if interval < std::time::Duration::from_secs(600) { + return Err(Error::Message( + "metadata_auto_snapshot_interval too small or negative".into(), + )); + } + + bg.spawn_worker(crate::snapshot::AutoSnapshotWorker::new( + self.clone(), + interval, + )); + } + + Ok(()) } - pub fn bucket_helper(&self) -> helper::bucket::BucketHelper { + pub fn bucket_helper(&self) -> helper::bucket::BucketHelper<'_> { helper::bucket::BucketHelper(self) } - pub fn key_helper(&self) -> helper::key::KeyHelper { + pub fn key_helper(&self) -> helper::key::KeyHelper<'_> { helper::key::KeyHelper(self) } + + pub async fn locked_helper(&self) -> helper::locked::LockedHelper<'_> { + let lock = self.bucket_lock.lock().await; + helper::locked::LockedHelper(self, Some(lock)) + } } #[cfg(feature = "k2v")] diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 576d03f3..e5506d7e 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -1,19 +1,15 @@ use std::time::Duration; -use garage_util::crdt::*; use garage_util::data::*; -use garage_util::error::{Error as GarageError, OkOrMessage}; +use garage_util::error::OkOrMessage; use garage_util::time::*; use garage_table::util::*; -use crate::bucket_alias_table::*; use crate::bucket_table::*; use crate::garage::Garage; use crate::helper::error::*; -use crate::helper::key::KeyHelper; use crate::key_table::*; -use crate::permission::BucketKeyPerm; use crate::s3::object_table::*; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -71,6 +67,49 @@ impl<'a> BucketHelper<'a> { } } + /// Find a bucket by its global alias or a prefix of its uuid + pub async fn admin_get_existing_matching_bucket( + &self, + pattern: &String, + ) -> Result { + if let Some(uuid) = self.resolve_global_bucket_name(pattern).await? { + return Ok(uuid); + } else if pattern.len() >= 2 { + let hexdec = pattern + .get(..pattern.len() & !1) + .and_then(|x| hex::decode(x).ok()); + if let Some(hex) = hexdec { + let mut start = [0u8; 32]; + start + .as_mut_slice() + .get_mut(..hex.len()) + .ok_or_bad_request("invalid length")? + .copy_from_slice(&hex); + let mut candidates = self + .0 + .bucket_table + .get_range( + &EmptyKey, + Some(start.into()), + Some(DeletedFilter::NotDeleted), + 10, + EnumerationOrder::Forward, + ) + .await? + .into_iter() + .collect::>(); + candidates.retain(|x| hex::encode(x.id).starts_with(pattern)); + if candidates.len() == 1 { + return Ok(candidates.into_iter().next().unwrap().id); + } + } + } + Err(Error::BadRequest(format!( + "Bucket not found / several matching buckets: {}", + pattern + ))) + } + /// Returns a Bucket if it is present in bucket table, /// even if it is in deleted state. Querying a non-existing /// bucket ID returns an internal error. @@ -96,341 +135,7 @@ impl<'a> BucketHelper<'a> { .ok_or_else(|| Error::NoSuchBucket(hex::encode(bucket_id))) } - /// Sets a new alias for a bucket in global namespace. - /// This function fails if: - /// - alias name is not valid according to S3 spec - /// - bucket does not exist or is deleted - /// - alias already exists and points to another bucket - pub async fn set_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - if !is_valid_bucket_name(alias_name) { - return Err(Error::InvalidBucketName(alias_name.to_string())); - } - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - - let alias = self.0.bucket_alias_table.get(&EmptyKey, alias_name).await?; - - if let Some(existing_alias) = alias.as_ref() { - if let Some(p_bucket) = existing_alias.state.get() { - if *p_bucket != bucket_id { - return Err(Error::BadRequest(format!( - "Alias {} already exists and points to different bucket: {:?}", - alias_name, p_bucket - ))); - } - } - } - - // Checks ok, add alias - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - - let alias_ts = increment_logical_clock_2( - bucket_p.aliases.get_timestamp(alias_name), - alias.as_ref().map(|a| a.state.timestamp()).unwrap_or(0), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - let alias = match alias { - None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)) - .ok_or_else(|| Error::InvalidBucketName(alias_name.clone()))?, - Some(mut a) => { - a.state = Lww::raw(alias_ts, Some(bucket_id)); - a - } - }; - self.0.bucket_alias_table.insert(&alias).await?; - - bucket_p.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, true); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Unsets an alias for a bucket in global namespace. - /// This function fails if: - /// - bucket does not exist or is deleted - /// - alias does not exist or maps to another bucket (-> internal error) - /// - bucket has no other aliases (global or local) - pub async fn unset_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut bucket_state = bucket.state.as_option_mut().unwrap(); - - let mut alias = self - .0 - .bucket_alias_table - .get(&EmptyKey, alias_name) - .await? - .filter(|a| a.state.get().map(|x| x == bucket_id).unwrap_or(false)) - .ok_or_message(format!( - "Internal error: alias not found or does not point to bucket {:?}", - bucket_id - ))?; - - let has_other_global_aliases = bucket_state - .aliases - .items() - .iter() - .any(|(name, _, active)| name != alias_name && *active); - let has_other_local_aliases = bucket_state - .local_aliases - .items() - .iter() - .any(|(_, _, active)| *active); - if !has_other_global_aliases && !has_other_local_aliases { - return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); - } - - // Checks ok, remove alias - let alias_ts = increment_logical_clock_2( - alias.state.timestamp(), - bucket_state.aliases.get_timestamp(alias_name), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - alias.state = Lww::raw(alias_ts, None); - self.0.bucket_alias_table.insert(&alias).await?; - - bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Ensures a bucket does not have a certain global alias. - /// Contrarily to unset_global_bucket_alias, this does not - /// fail on any condition other than: - /// - bucket cannot be found (its fine if it is in deleted state) - /// - alias cannot be found (its fine if it points to nothing or - /// to another bucket) - pub async fn purge_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - let mut bucket = self.get_internal_bucket(bucket_id).await?; - - let mut alias = self - .0 - .bucket_alias_table - .get(&EmptyKey, alias_name) - .await? - .ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?; - - // Checks ok, remove alias - let alias_ts = match bucket.state.as_option() { - Some(bucket_state) => increment_logical_clock_2( - alias.state.timestamp(), - bucket_state.aliases.get_timestamp(alias_name), - ), - None => increment_logical_clock(alias.state.timestamp()), - }; - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - if alias.state.get() == &Some(bucket_id) { - alias.state = Lww::raw(alias_ts, None); - self.0.bucket_alias_table.insert(&alias).await?; - } - - if let Some(mut bucket_state) = bucket.state.as_option_mut() { - bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - } - - Ok(()) - } - - /// Sets a new alias for a bucket in the local namespace of a key. - /// This function fails if: - /// - alias name is not valid according to S3 spec - /// - bucket does not exist or is deleted - /// - key does not exist or is deleted - /// - alias already exists and points to another bucket - pub async fn set_local_bucket_alias( - &self, - bucket_id: Uuid, - key_id: &String, - alias_name: &String, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - if !is_valid_bucket_name(alias_name) { - return Err(Error::InvalidBucketName(alias_name.to_string())); - } - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = key_helper.get_existing_key(key_id).await?; - - let mut key_param = key.state.as_option_mut().unwrap(); - - if let Some(Some(existing_alias)) = key_param.local_aliases.get(alias_name) { - if *existing_alias != bucket_id { - return Err(Error::BadRequest(format!("Alias {} already exists in namespace of key {} and points to different bucket: {:?}", alias_name, key.key_id, existing_alias))); - } - } - - // Checks ok, add alias - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); - - // Calculate the timestamp to assign to this aliasing in the two local_aliases maps - // (the one from key to bucket, and the reverse one stored in the bucket iself) - // so that merges on both maps in case of a concurrent operation resolve - // to the same alias being set - let alias_ts = increment_logical_clock_2( - key_param.local_aliases.get_timestamp(alias_name), - bucket_p - .local_aliases - .get_timestamp(&bucket_p_local_alias_key), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id)); - self.0.key_table.insert(&key).await?; - - bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, true); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Unsets an alias for a bucket in the local namespace of a key. - /// This function fails if: - /// - bucket does not exist or is deleted - /// - key does not exist or is deleted - /// - alias does not exist or maps to another bucket (-> internal error) - /// - bucket has no other aliases (global or local) - pub async fn unset_local_bucket_alias( - &self, - bucket_id: Uuid, - key_id: &String, - alias_name: &String, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = key_helper.get_existing_key(key_id).await?; - - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - - if key - .state - .as_option() - .unwrap() - .local_aliases - .get(alias_name) - .cloned() - .flatten() != Some(bucket_id) - { - return Err(GarageError::Message(format!( - "Bucket {:?} does not have alias {} in namespace of key {}", - bucket_id, alias_name, key_id - )) - .into()); - } - - let has_other_global_aliases = bucket_p - .aliases - .items() - .iter() - .any(|(_, _, active)| *active); - let has_other_local_aliases = bucket_p - .local_aliases - .items() - .iter() - .any(|((k, n), _, active)| *k == key.key_id && n == alias_name && *active); - if !has_other_global_aliases && !has_other_local_aliases { - return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); - } - - // Checks ok, remove alias - let mut key_param = key.state.as_option_mut().unwrap(); - let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); - - let alias_ts = increment_logical_clock_2( - key_param.local_aliases.get_timestamp(alias_name), - bucket_p - .local_aliases - .get_timestamp(&bucket_p_local_alias_key), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None); - self.0.key_table.insert(&key).await?; - - bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Sets permissions for a key on a bucket. - /// This function fails if: - /// - bucket or key cannot be found at all (its ok if they are in deleted state) - /// - bucket or key is in deleted state and we are trying to set permissions other than "deny - /// all" - pub async fn set_bucket_key_permissions( - &self, - bucket_id: Uuid, - key_id: &String, - mut perm: BucketKeyPerm, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - let mut bucket = self.get_internal_bucket(bucket_id).await?; - let mut key = key_helper.get_internal_key(key_id).await?; - - if let Some(bstate) = bucket.state.as_option() { - if let Some(kp) = bstate.authorized_keys.get(key_id) { - perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp); - } - } else if perm.is_any() { - return Err(Error::BadRequest( - "Trying to give permissions on a deleted bucket".into(), - )); - } - - if let Some(kstate) = key.state.as_option() { - if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) { - perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp); - } - } else if perm.is_any() { - return Err(Error::BadRequest( - "Trying to give permissions to a deleted key".into(), - )); - } - - // ---- timestamp-ensured causality barrier ---- - - if let Some(bstate) = bucket.state.as_option_mut() { - bstate.authorized_keys = Map::put_mutator(key_id.clone(), perm); - self.0.bucket_table.insert(&bucket).await?; - } - - if let Some(kstate) = key.state.as_option_mut() { - kstate.authorized_buckets = Map::put_mutator(bucket_id, perm); - self.0.key_table.insert(&key).await?; - } - - Ok(()) - } + // ---- pub async fn is_bucket_empty(&self, bucket_id: Uuid) -> Result { let objects = self @@ -450,10 +155,12 @@ impl<'a> BucketHelper<'a> { #[cfg(feature = "k2v")] { - use garage_rpc::ring::Ring; - use std::sync::Arc; - - let ring: Arc = self.0.system.ring.borrow().clone(); + let node_id_vec = self + .0 + .system + .cluster_layout() + .all_nongateway_nodes() + .to_vec(); let k2vindexes = self .0 .k2v @@ -462,7 +169,7 @@ impl<'a> BucketHelper<'a> { .get_range( &bucket_id, None, - Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())), + Some((DeletedFilter::NotDeleted, node_id_vec)), 10, EnumerationOrder::Forward, ) diff --git a/src/model/helper/error.rs b/src/model/helper/error.rs index 3ca8f55c..bc483c7d 100644 --- a/src/model/helper/error.rs +++ b/src/model/helper/error.rs @@ -1,30 +1,30 @@ -use err_derive::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; use garage_util::error::Error as GarageError; #[derive(Debug, Error, Serialize, Deserialize)] pub enum Error { - #[error(display = "Internal error: {}", _0)] - Internal(#[error(source)] GarageError), + #[error("Internal error: {0}")] + Internal(#[from] GarageError), - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), } -impl From for Error { - fn from(e: netapp::error::Error) -> Self { - Error::Internal(GarageError::Netapp(e)) +impl From for Error { + fn from(e: garage_net::error::Error) -> Self { + Error::Internal(GarageError::Net(e)) } } diff --git a/src/model/helper/key.rs b/src/model/helper/key.rs index c1a8e974..b8a99d55 100644 --- a/src/model/helper/key.rs +++ b/src/model/helper/key.rs @@ -1,12 +1,9 @@ use garage_table::util::*; -use garage_util::crdt::*; use garage_util::error::OkOrMessage; use crate::garage::Garage; -use crate::helper::bucket::BucketHelper; use crate::helper::error::*; use crate::key_table::{Key, KeyFilter}; -use crate::permission::BucketKeyPerm; pub struct KeyHelper<'a>(pub(crate) &'a Garage); @@ -65,38 +62,4 @@ impl<'a> KeyHelper<'a> { Ok(candidates.into_iter().next().unwrap()) } } - - /// Deletes an API access key - pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> { - let bucket_helper = BucketHelper(self.0); - - let state = key.state.as_option_mut().unwrap(); - - // --- done checking, now commit --- - // (the step at unset_local_bucket_alias will fail if a bucket - // does not have another alias, the deletion will be - // interrupted in the middle if that happens) - - // 1. Delete local aliases - for (alias, _, to) in state.local_aliases.items().iter() { - if let Some(bucket_id) = to { - bucket_helper - .unset_local_bucket_alias(*bucket_id, &key.key_id, alias) - .await?; - } - } - - // 2. Remove permissions on all authorized buckets - for (ab_id, _auth) in state.authorized_buckets.items().iter() { - bucket_helper - .set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) - .await?; - } - - // 3. Actually delete key - key.state = Deletable::delete(); - self.0.key_table.insert(key).await?; - - Ok(()) - } } diff --git a/src/model/helper/locked.rs b/src/model/helper/locked.rs new file mode 100644 index 00000000..98344b63 --- /dev/null +++ b/src/model/helper/locked.rs @@ -0,0 +1,654 @@ +use std::collections::{HashMap, HashSet}; + +use garage_db as db; + +use garage_util::crdt::*; +use garage_util::data::*; +use garage_util::error::{Error as GarageError, OkOrMessage}; +use garage_util::time::*; + +use garage_table::util::*; + +use crate::bucket_alias_table::*; +use crate::garage::Garage; +use crate::helper::bucket::BucketHelper; +use crate::helper::error::*; +use crate::helper::key::KeyHelper; +use crate::key_table::*; +use crate::permission::BucketKeyPerm; + +/// A LockedHelper is the mandatory struct to hold when doing operations +/// that modify access keys or bucket aliases. This structure takes +/// a lock to a unit value that is in the globally-shared Garage struct. +/// +/// This avoid several concurrent requests to modify the list of buckets +/// and aliases at the same time, ending up in inconsistent states. +/// This DOES NOT FIX THE FUNDAMENTAL ISSUE as CreateBucket requests handled +/// by different API nodes can still break the cluster, but it is a first +/// fix that allows consistency to be maintained if all such requests are +/// directed to a single node, which is doable for many deployments. +/// +/// See issues: #649, #723 +pub struct LockedHelper<'a>( + pub(crate) &'a Garage, + pub(crate) Option>, +); + +impl<'a> Drop for LockedHelper<'a> { + fn drop(&mut self) { + // make it explicit that the mutexguard lives until here + drop(self.1.take()) + } +} + +#[allow(clippy::ptr_arg)] +impl<'a> LockedHelper<'a> { + pub fn bucket(&self) -> BucketHelper<'a> { + BucketHelper(self.0) + } + + pub fn key(&self) -> KeyHelper<'a> { + KeyHelper(self.0) + } + + // ================================================ + // global bucket aliases + // ================================================ + + /// Sets a new alias for a bucket in global namespace. + /// This function fails if: + /// - alias name is not valid according to S3 spec + /// - bucket does not exist or is deleted + /// - alias already exists and points to another bucket + pub async fn set_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) { + return Err(Error::InvalidBucketName(alias_name.to_string())); + } + + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + + let alias = self.0.bucket_alias_table.get(&EmptyKey, alias_name).await?; + + if let Some(existing_alias) = alias.as_ref() { + if let Some(p_bucket) = existing_alias.state.get() { + if *p_bucket != bucket_id { + return Err(Error::BadRequest(format!( + "Alias {} already exists and points to different bucket: {:?}", + alias_name, p_bucket + ))); + } + } + } + + // Checks ok, add alias + let bucket_p = bucket.state.as_option_mut().unwrap(); + + let alias_ts = increment_logical_clock_2( + bucket_p.aliases.get_timestamp(alias_name), + alias.as_ref().map(|a| a.state.timestamp()).unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + let alias = match alias { + None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)), + Some(mut a) => { + a.state = Lww::raw(alias_ts, Some(bucket_id)); + a + } + }; + self.0.bucket_alias_table.insert(&alias).await?; + + bucket_p.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, true); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Unsets an alias for a bucket in global namespace. + /// This function fails if: + /// - bucket does not exist or is deleted + /// - alias does not exist or maps to another bucket (-> internal error) + /// - bucket has no other aliases (global or local) + pub async fn unset_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let bucket_state = bucket.state.as_option_mut().unwrap(); + + let mut alias = self + .0 + .bucket_alias_table + .get(&EmptyKey, alias_name) + .await? + .filter(|a| a.state.get().map(|x| x == bucket_id).unwrap_or(false)) + .ok_or_message(format!( + "Internal error: alias not found or does not point to bucket {:?}", + bucket_id + ))?; + + let has_other_global_aliases = bucket_state + .aliases + .items() + .iter() + .any(|(name, _, active)| name != alias_name && *active); + let has_other_local_aliases = bucket_state + .local_aliases + .items() + .iter() + .any(|(_, _, active)| *active); + if !has_other_global_aliases && !has_other_local_aliases { + return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); + } + + // Checks ok, remove alias + let alias_ts = increment_logical_clock_2( + alias.state.timestamp(), + bucket_state.aliases.get_timestamp(alias_name), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + alias.state = Lww::raw(alias_ts, None); + self.0.bucket_alias_table.insert(&alias).await?; + + bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Ensures a bucket does not have a certain global alias. + /// Contrarily to unset_global_bucket_alias, this does not + /// fail on any condition other than: + /// - bucket cannot be found (its fine if it is in deleted state) + /// - alias cannot be found (its fine if it points to nothing or + /// to another bucket) + pub async fn purge_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + + let mut alias = self + .0 + .bucket_alias_table + .get(&EmptyKey, alias_name) + .await? + .ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?; + + // Checks ok, remove alias + let alias_ts = increment_logical_clock_2( + alias.state.timestamp(), + bucket + .state + .as_option() + .map(|p| p.aliases.get_timestamp(alias_name)) + .unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + if alias.state.get() == &Some(bucket_id) { + alias.state = Lww::raw(alias_ts, None); + self.0.bucket_alias_table.insert(&alias).await?; + } + + if let Some(bucket_state) = bucket.state.as_option_mut() { + bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + } + + Ok(()) + } + + // ================================================ + // local bucket aliases + // ================================================ + + /// Sets a new alias for a bucket in the local namespace of a key. + /// This function fails if: + /// - alias name is not valid according to S3 spec + /// - bucket does not exist or is deleted + /// - key does not exist or is deleted + /// - alias already exists and points to another bucket + pub async fn set_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) { + return Err(Error::InvalidBucketName(alias_name.to_string())); + } + + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let mut key = self.key().get_existing_key(key_id).await?; + + let key_param = key.state.as_option_mut().unwrap(); + + if let Some(Some(existing_alias)) = key_param.local_aliases.get(alias_name) { + if *existing_alias != bucket_id { + return Err(Error::BadRequest(format!("Alias {} already exists in namespace of key {} and points to different bucket: {:?}", alias_name, key.key_id, existing_alias))); + } + } + + // Checks ok, add alias + let bucket_p = bucket.state.as_option_mut().unwrap(); + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + // Calculate the timestamp to assign to this aliasing in the two local_aliases maps + // (the one from key to bucket, and the reverse one stored in the bucket itself) + // so that merges on both maps in case of a concurrent operation resolve + // to the same alias being set + let alias_ts = increment_logical_clock_2( + key_param.local_aliases.get_timestamp(alias_name), + bucket_p + .local_aliases + .get_timestamp(&bucket_p_local_alias_key), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id)); + self.0.key_table.insert(&key).await?; + + bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, true); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Unsets an alias for a bucket in the local namespace of a key. + /// This function fails if: + /// - bucket does not exist or is deleted + /// - key does not exist or is deleted + /// - alias does not exist or maps to another bucket (-> internal error) + /// - bucket has no other aliases (global or local) + pub async fn unset_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let mut key = self.key().get_existing_key(key_id).await?; + + let key_p = key.state.as_option().unwrap(); + let bucket_p = bucket.state.as_option_mut().unwrap(); + + if key_p.local_aliases.get(alias_name).cloned().flatten() != Some(bucket_id) { + return Err(GarageError::Message(format!( + "Bucket {:?} does not have alias {} in namespace of key {}", + bucket_id, alias_name, key_id + )) + .into()); + } + + let has_other_global_aliases = bucket_p + .aliases + .items() + .iter() + .any(|(_, _, active)| *active); + let has_other_local_aliases = bucket_p + .local_aliases + .items() + .iter() + .any(|((k, n), _, active)| (*k != key.key_id || n != alias_name) && *active); + + if !has_other_global_aliases && !has_other_local_aliases { + return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); + } + + // Checks ok, remove alias + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + let alias_ts = increment_logical_clock_2( + key_p.local_aliases.get_timestamp(alias_name), + bucket_p + .local_aliases + .get_timestamp(&bucket_p_local_alias_key), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + key.state.as_option_mut().unwrap().local_aliases = + LwwMap::raw_item(alias_name.clone(), alias_ts, None); + self.0.key_table.insert(&key).await?; + + bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Ensures a bucket does not have a certain local alias. + /// Contrarily to unset_local_bucket_alias, this does not + /// fail on any condition other than: + /// - bucket cannot be found (its fine if it is in deleted state) + /// - key cannot be found (its fine if alias in key points to nothing + /// or to another bucket) + pub async fn purge_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + let mut key = self.key().get_internal_key(key_id).await?; + + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + let alias_ts = increment_logical_clock_2( + key.state + .as_option() + .map(|p| p.local_aliases.get_timestamp(alias_name)) + .unwrap_or(0), + bucket + .state + .as_option() + .map(|p| p.local_aliases.get_timestamp(&bucket_p_local_alias_key)) + .unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + if let Some(kp) = key.state.as_option_mut() { + kp.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None); + self.0.key_table.insert(&key).await?; + } + + if let Some(bp) = bucket.state.as_option_mut() { + bp.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + } + + Ok(()) + } + + // ================================================ + // permissions + // ================================================ + + /// Sets permissions for a key on a bucket. + /// This function fails if: + /// - bucket or key cannot be found at all (its ok if they are in deleted state) + /// - bucket or key is in deleted state and we are trying to set + /// permissions other than "deny all" + pub async fn set_bucket_key_permissions( + &self, + bucket_id: Uuid, + key_id: &String, + mut perm: BucketKeyPerm, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + let mut key = self.key().get_internal_key(key_id).await?; + + if let Some(bstate) = bucket.state.as_option() { + if let Some(kp) = bstate.authorized_keys.get(key_id) { + perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp); + } + } else if perm.is_any() { + return Err(Error::BadRequest( + "Trying to give permissions on a deleted bucket".into(), + )); + } + + if let Some(kstate) = key.state.as_option() { + if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) { + perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp); + } + } else if perm.is_any() { + return Err(Error::BadRequest( + "Trying to give permissions to a deleted key".into(), + )); + } + + // ---- timestamp-ensured causality barrier ---- + + if let Some(bstate) = bucket.state.as_option_mut() { + bstate.authorized_keys = Map::put_mutator(key_id.clone(), perm); + self.0.bucket_table.insert(&bucket).await?; + } + + if let Some(kstate) = key.state.as_option_mut() { + kstate.authorized_buckets = Map::put_mutator(bucket_id, perm); + self.0.key_table.insert(&key).await?; + } + + Ok(()) + } + + // ================================================ + // keys + // ================================================ + + /// Deletes an API access key + pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> { + let state = key.state.as_option_mut().unwrap(); + + // --- done checking, now commit --- + + // 1. Delete local aliases + for (alias, _, to) in state.local_aliases.items().iter() { + if let Some(bucket_id) = to { + self.purge_local_bucket_alias(*bucket_id, &key.key_id, alias) + .await?; + } + } + + // 2. Remove permissions on all authorized buckets + for (ab_id, _auth) in state.authorized_buckets.items().iter() { + self.set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) + .await?; + } + + // 3. Actually delete key + key.state = Deletable::delete(); + self.0.key_table.insert(key).await?; + + Ok(()) + } + + // ================================================ + // repair procedure + // ================================================ + + pub async fn repair_aliases(&self) -> Result<(), GarageError> { + self.0.db.transaction(|tx| { + info!("--- begin repair_aliases transaction ----"); + + // 1. List all non-deleted buckets, so that we can fix bad aliases + let mut all_buckets: HashSet = HashSet::new(); + + for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? { + let bucket = self + .0 + .bucket_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + if !bucket.is_deleted() { + all_buckets.insert(bucket.id); + } + } + + info!("number of buckets: {}", all_buckets.len()); + + // 2. List all aliases declared in bucket_alias_table and key_table + // Take note of aliases that point to non-existing buckets + let mut global_aliases: HashMap = HashMap::new(); + + { + let mut delete_global = vec![]; + for item in tx.range::<&[u8], _>(&self.0.bucket_alias_table.data.store, ..)? { + let mut alias = self + .0 + .bucket_alias_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + if let Some(id) = alias.state.get() { + if all_buckets.contains(id) { + // keep aliases + global_aliases.insert(alias.name().to_string(), *id); + } else { + // delete alias + warn!( + "global alias: remove {} -> {:?} (bucket is deleted)", + alias.name(), + id + ); + alias.state.update(None); + delete_global.push(alias); + } + } + } + + info!("number of global aliases: {}", global_aliases.len()); + + info!("global alias table: {} entries fixed", delete_global.len()); + for ga in delete_global { + debug!("Enqueue update to global alias table: {:?}", ga); + self.0.bucket_alias_table.queue_insert(tx, &ga)?; + } + } + + let mut local_aliases: HashMap<(String, String), Uuid> = HashMap::new(); + + { + let mut delete_local = vec![]; + + for item in tx.range::<&[u8], _>(&self.0.key_table.data.store, ..)? { + let mut key = self + .0 + .key_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + let Some(p) = key.state.as_option_mut() else { + continue; + }; + let mut has_changes = false; + for (name, _, to) in p.local_aliases.items().to_vec() { + if let Some(id) = to { + if all_buckets.contains(&id) { + local_aliases.insert((key.key_id.clone(), name), id); + } else { + warn!( + "local alias: remove ({}, {}) -> {:?} (bucket is deleted)", + key.key_id, name, id + ); + p.local_aliases.update_in_place(name, None); + has_changes = true; + } + } + } + if has_changes { + delete_local.push(key); + } + } + + info!("number of local aliases: {}", local_aliases.len()); + + info!("key table: {} entries fixed", delete_local.len()); + for la in delete_local { + debug!("Enqueue update to key table: {:?}", la); + self.0.key_table.queue_insert(tx, &la)?; + } + } + + // 4. Reverse the alias maps to determine the aliases per-bucket + let mut bucket_global: HashMap> = HashMap::new(); + let mut bucket_local: HashMap> = HashMap::new(); + + for (name, bucket) in global_aliases { + bucket_global.entry(bucket).or_default().push(name); + } + for ((key, name), bucket) in local_aliases { + bucket_local.entry(bucket).or_default().push((key, name)); + } + + // 5. Fix the bucket table to ensure consistency + let mut bucket_updates = vec![]; + + for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? { + let bucket = self + .0 + .bucket_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + let mut bucket2 = bucket.clone(); + let Some(param) = bucket2.state.as_option_mut() else { + continue; + }; + + // fix global aliases + { + let ga = bucket_global.remove(&bucket.id).unwrap_or_default(); + for (name, _, active) in param.aliases.items().to_vec() { + if active && !ga.contains(&name) { + warn!("bucket {:?}: remove global alias {}", bucket.id, name); + param.aliases.update_in_place(name, false); + } + } + for name in ga { + if param.aliases.get(&name).copied() != Some(true) { + warn!("bucket {:?}: add global alias {}", bucket.id, name); + param.aliases.update_in_place(name, true); + } + } + } + + // fix local aliases + { + let la = bucket_local.remove(&bucket.id).unwrap_or_default(); + for (pair, _, active) in param.local_aliases.items().to_vec() { + if active && !la.contains(&pair) { + warn!("bucket {:?}: remove local alias {:?}", bucket.id, pair); + param.local_aliases.update_in_place(pair, false); + } + } + for pair in la { + if param.local_aliases.get(&pair).copied() != Some(true) { + warn!("bucket {:?}: add local alias {:?}", bucket.id, pair); + param.local_aliases.update_in_place(pair, true); + } + } + } + + if bucket2 != bucket { + bucket_updates.push(bucket2); + } + } + + info!("bucket table: {} entries fixed", bucket_updates.len()); + for b in bucket_updates { + debug!("Enqueue update to bucket table: {:?}", b); + self.0.bucket_table.queue_insert(tx, &b)?; + } + + info!("--- end repair_aliases transaction ----"); + + Ok(()) + })?; + + info!("repair_aliases is done"); + + Ok(()) + } +} diff --git a/src/model/helper/mod.rs b/src/model/helper/mod.rs index dd947c86..5534ae3d 100644 --- a/src/model/helper/mod.rs +++ b/src/model/helper/mod.rs @@ -1,3 +1,4 @@ pub mod bucket; pub mod error; pub mod key; +pub mod locked; diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index a46c165f..aa13ee7b 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use garage_db as db; -use garage_rpc::ring::Ring; +use garage_rpc::layout::LayoutHelper; use garage_rpc::system::System; use garage_util::background::BackgroundRunner; use garage_util::data::*; @@ -83,9 +83,9 @@ impl Entry for CounterEntry { } impl CounterEntry { - pub fn filtered_values(&self, ring: &Ring) -> HashMap { - let nodes = &ring.layout.node_id_vec[..]; - self.filtered_values_with_nodes(nodes) + pub fn filtered_values(&self, layout: &LayoutHelper) -> HashMap { + let nodes = layout.all_nongateway_nodes(); + self.filtered_values_with_nodes(&nodes) } pub fn filtered_values_with_nodes(&self, nodes: &[Uuid]) -> HashMap { @@ -232,7 +232,7 @@ impl IndexCounter { let now = now_msec(); for (s, inc) in counts.iter() { - let mut ent = entry.values.entry(s.to_string()).or_insert((0, 0)); + let ent = entry.values.entry(s.to_string()).or_insert((0, 0)); ent.0 = std::cmp::max(ent.0 + 1, now); ent.1 += *inc; } @@ -348,7 +348,7 @@ impl IndexCounter { }, }; for (s, v) in counts.iter() { - let mut tv = local_counter.values.entry(s.to_string()).or_insert((0, 0)); + let tv = local_counter.values.entry(s.to_string()).or_insert((0, 0)); tv.0 = std::cmp::max(tv.0 + 1, now); tv.1 += v; } diff --git a/src/model/k2v/causality.rs b/src/model/k2v/causality.rs index c80ebd39..7d311ede 100644 --- a/src/model/k2v/causality.rs +++ b/src/model/k2v/causality.rs @@ -16,8 +16,6 @@ use serde::{Deserialize, Serialize}; use garage_util::data::*; -use crate::helper::error::{Error as HelperError, OkOrBadRequest}; - /// Node IDs used in K2V are u64 integers that are the abbreviation /// of full Garage node IDs which are 256-bit UUIDs. pub type K2VNodeId = u64; @@ -99,10 +97,6 @@ impl CausalContext { Some(ret) } - pub fn parse_helper(s: &str) -> Result { - Self::parse(s).ok_or_bad_request("Invalid causality token") - } - /// Check if this causal context contains newer items than another one pub fn is_newer_than(&self, other: &Self) -> bool { vclock_gt(&self.vector_clock, &other.vector_clock) diff --git a/src/model/k2v/rpc.rs b/src/model/k2v/rpc.rs index 37e142f6..821f4549 100644 --- a/src/model/k2v/rpc.rs +++ b/src/model/k2v/rpc.rs @@ -10,7 +10,6 @@ use std::convert::TryInto; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; -use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::StreamExt; use serde::{Deserialize, Serialize}; @@ -127,23 +126,21 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&partition.hash()); + .storage_nodes(&partition.hash()); who.sort(); self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, K2VRpc::InsertItem(InsertedItem { partition, sort_key, causal_context, value, }), - RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(1) - .interrupt_after_quorum(true), + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(1), ) .await?; @@ -168,7 +165,7 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&partition.hash()); + .storage_nodes(&partition.hash()); who.sort(); call_list.entry(who).or_default().push(InsertedItem { @@ -187,14 +184,12 @@ impl K2VRpcHandler { let call_futures = call_list.into_iter().map(|(nodes, items)| async move { let resp = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, &nodes[..], K2VRpc::InsertManyItems(items), - RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(1) - .interrupt_after_quorum(true), + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(1), ) .await?; Ok::<_, Error>((nodes, resp)) @@ -227,11 +222,11 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&poll_key.partition.hash()); + .storage_nodes(&poll_key.partition.hash()); - let rpc = self.system.rpc.try_call_many( + let rpc = self.system.rpc_helper().try_call_many( &self.endpoint, - &nodes[..], + &nodes, K2VRpc::PollItem { key: poll_key, causal_context, @@ -239,9 +234,10 @@ impl K2VRpcHandler { }, RequestStrategy::with_priority(PRIO_NORMAL) .with_quorum(self.item_table.data.replication.read_quorum()) + .send_all_at_once(true) .without_timeout(), ); - let timeout_duration = Duration::from_millis(timeout_msec) + self.system.rpc.rpc_timeout(); + let timeout_duration = Duration::from_millis(timeout_msec); let resps = select! { r = rpc => r?, _ = tokio::time::sleep(timeout_duration) => return Ok(None), @@ -287,7 +283,7 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&range.partition.hash()); + .storage_nodes(&range.partition.hash()); let quorum = self.item_table.data.replication.read_quorum(); let msg = K2VRpc::PollRange { range, @@ -296,11 +292,15 @@ impl K2VRpcHandler { }; // Send the request to all nodes, use FuturesUnordered to get the responses in any order - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; let rs = RequestStrategy::with_priority(PRIO_NORMAL).without_timeout(); let mut requests = nodes .iter() - .map(|node| self.system.rpc.call(&self.endpoint, *node, msg.clone(), rs)) + .map(|node| { + self.system + .rpc_helper() + .call(&self.endpoint, *node, msg.clone(), rs.clone()) + }) .collect::>(); // Fetch responses. This procedure stops fetching responses when any of the following @@ -309,15 +309,14 @@ impl K2VRpcHandler { // - we have a response to a read quorum of requests (e.g. 2/3), and an extra delay // has passed since the quorum was achieved // - a global RPC timeout expired - // The extra delay after a quorum was received is usefull if the third response was to + // The extra delay after a quorum was received is useful if the third response was to // arrive during this short interval: this would allow us to consider all the data seen // by that last node in the response we produce, and would likely help reduce the // size of the seen marker that we will return (because we would have an info of the // kind: all items produced by that node until time ts have been returned, so we can // bump the entry in the global vector clock and possibly remove some item-specific // vector clocks) - let mut deadline = - Instant::now() + Duration::from_millis(timeout_msec) + self.system.rpc.rpc_timeout(); + let mut deadline = Instant::now() + Duration::from_millis(timeout_msec); let mut resps = vec![]; let mut errors = vec![]; loop { @@ -339,7 +338,7 @@ impl K2VRpcHandler { } if errors.len() > nodes.len() - quorum { let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - return Err(Error::Quorum(quorum, resps.len(), nodes.len(), errors).into()); + return Err(Error::Quorum(quorum, None, resps.len(), nodes.len(), errors).into()); } // Take all returned items into account to produce the response. @@ -500,7 +499,7 @@ impl K2VRpcHandler { } else { // If no seen marker was specified, we do not poll for anything. // We return immediately with the set of known items (even if - // it is empty), which will give the client an inital view of + // it is empty), which will give the client an initial view of // the dataset and an initial seen marker for further // PollRange calls. self.poll_range_read_range(range, &RangeSeenMarker::default()) @@ -537,7 +536,6 @@ impl K2VRpcHandler { } } -#[async_trait] impl EndpointHandler for K2VRpcHandler { async fn handle(self: &Arc, message: &K2VRpc, _from: NodeID) -> Result { match message { diff --git a/src/model/key_table.rs b/src/model/key_table.rs index a9762f1b..efb95f08 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -7,48 +7,7 @@ use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema}; use crate::permission::BucketKeyPerm; -pub(crate) mod v05 { - use garage_util::crdt; - use serde::{Deserialize, Serialize}; - - /// An api key - #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct Key { - /// The id of the key (immutable), used as partition key - pub key_id: String, - - /// The secret_key associated - pub secret_key: String, - - /// Name for the key - pub name: crdt::Lww, - - /// Is the key deleted - pub deleted: crdt::Bool, - - /// Buckets in which the key is authorized. Empty if `Key` is deleted - // CRDT interaction: deleted implies authorized_buckets is empty - pub authorized_buckets: crdt::LwwMap, - } - - /// Permission given to a key in a bucket - #[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct PermissionSet { - /// The key can be used to read the bucket - pub allow_read: bool, - /// The key can be used to write in the bucket - pub allow_write: bool, - } - - impl crdt::AutoCrdt for PermissionSet { - const WARN_IF_DIFFERENT: bool = true; - } - - impl garage_util::migrate::InitialFormat for Key {} -} - mod v08 { - use super::v05; use crate::permission::BucketKeyPerm; use garage_util::crdt; use garage_util::data::Uuid; @@ -86,32 +45,7 @@ mod v08 { pub local_aliases: crdt::LwwMap>, } - impl garage_util::migrate::Migrate for Key { - type Previous = v05::Key; - - fn migrate(old_k: v05::Key) -> Key { - let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone()); - - let state = if old_k.deleted.get() { - crdt::Deletable::Deleted - } else { - // Authorized buckets is ignored here, - // migration is performed in specific migration code in - // garage/migrate.rs - crdt::Deletable::Present(KeyParams { - secret_key: old_k.secret_key, - name, - allow_create_bucket: crdt::Lww::new(false), - authorized_buckets: crdt::Map::new(), - local_aliases: crdt::LwwMap::new(), - }) - }; - Key { - key_id: old_k.key_id, - state, - } - } - } + impl garage_util::migrate::InitialFormat for Key {} } pub use v08::*; diff --git a/src/model/lib.rs b/src/model/lib.rs index 4f20ea46..1939a7a9 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -1,9 +1,6 @@ #[macro_use] extern crate tracing; -// For migration from previous versions -pub(crate) mod prev; - pub mod permission; pub mod index_counter; @@ -18,4 +15,4 @@ pub mod s3; pub mod garage; pub mod helper; -pub mod migrate; +pub mod snapshot; diff --git a/src/model/migrate.rs b/src/model/migrate.rs deleted file mode 100644 index 4c74b43b..00000000 --- a/src/model/migrate.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::sync::Arc; - -use garage_util::crdt::*; -use garage_util::data::*; -use garage_util::encode::nonversioned_decode; -use garage_util::error::Error as GarageError; -use garage_util::time::*; - -use crate::prev::v051::bucket_table as old_bucket; - -use crate::bucket_alias_table::*; -use crate::bucket_table::*; -use crate::garage::Garage; -use crate::helper::error::*; -use crate::permission::*; - -pub struct Migrate { - pub garage: Arc, -} - -impl Migrate { - pub async fn migrate_buckets050(&self) -> Result<(), Error> { - let tree = self - .garage - .db - .open_tree("bucket:table") - .map_err(GarageError::from)?; - - let mut old_buckets = vec![]; - for res in tree.iter().map_err(GarageError::from)? { - let (_k, v) = res.map_err(GarageError::from)?; - let bucket = - nonversioned_decode::(&v[..]).map_err(GarageError::from)?; - old_buckets.push(bucket); - } - - for bucket in old_buckets { - if let old_bucket::BucketState::Present(p) = bucket.state.get() { - self.migrate_buckets050_do_bucket(&bucket, p).await?; - } - } - - Ok(()) - } - - pub async fn migrate_buckets050_do_bucket( - &self, - old_bucket: &old_bucket::Bucket, - old_bucket_p: &old_bucket::BucketParams, - ) -> Result<(), Error> { - let bucket_id = blake2sum(old_bucket.name.as_bytes()); - - let new_name = if is_valid_bucket_name(&old_bucket.name) { - old_bucket.name.clone() - } else { - // if old bucket name was not valid, replace it by - // a hex-encoded name derived from its identifier - hex::encode(&bucket_id.as_slice()[..16]) - }; - - let website = if *old_bucket_p.website.get() { - Some(WebsiteConfig { - index_document: "index.html".into(), - error_document: None, - }) - } else { - None - }; - - self.garage - .bucket_table - .insert(&Bucket { - id: bucket_id, - state: Deletable::Present(BucketParams { - creation_date: now_msec(), - authorized_keys: Map::new(), - aliases: LwwMap::new(), - local_aliases: LwwMap::new(), - website_config: Lww::new(website), - cors_config: Lww::new(None), - lifecycle_config: Lww::new(None), - quotas: Lww::new(Default::default()), - }), - }) - .await?; - - self.garage - .bucket_helper() - .set_global_bucket_alias(bucket_id, &new_name) - .await?; - - for (k, ts, perm) in old_bucket_p.authorized_keys.items().iter() { - self.garage - .bucket_helper() - .set_bucket_key_permissions( - bucket_id, - k, - BucketKeyPerm { - timestamp: *ts, - allow_read: perm.allow_read, - allow_write: perm.allow_write, - allow_owner: false, - }, - ) - .await?; - } - - Ok(()) - } -} diff --git a/src/model/prev/mod.rs b/src/model/prev/mod.rs deleted file mode 100644 index 68bb1502..00000000 --- a/src/model/prev/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod v051; diff --git a/src/model/prev/v051/bucket_table.rs b/src/model/prev/v051/bucket_table.rs deleted file mode 100644 index 19893458..00000000 --- a/src/model/prev/v051/bucket_table.rs +++ /dev/null @@ -1,63 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use garage_table::crdt::Crdt; -use garage_table::*; - -use crate::key_table::v05::PermissionSet; - -/// A bucket is a collection of objects -/// -/// Its parameters are not directly accessible as: -/// - It must be possible to merge paramaters, hence the use of a LWW CRDT. -/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub struct Bucket { - /// Name of the bucket - pub name: String, - /// State, and configuration if not deleted, of the bucket - pub state: crdt::Lww, -} - -/// State of a bucket -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub enum BucketState { - /// The bucket is deleted - Deleted, - /// The bucket exists - Present(BucketParams), -} - -impl Crdt for BucketState { - fn merge(&mut self, o: &Self) { - match o { - BucketState::Deleted => *self = BucketState::Deleted, - BucketState::Present(other_params) => { - if let BucketState::Present(params) = self { - params.merge(other_params); - } - } - } - } -} - -/// Configuration for a bucket -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub struct BucketParams { - /// Map of key with access to the bucket, and what kind of access they give - pub authorized_keys: crdt::LwwMap, - /// Is the bucket served as http - pub website: crdt::Lww, -} - -impl Crdt for BucketParams { - fn merge(&mut self, o: &Self) { - self.authorized_keys.merge(&o.authorized_keys); - self.website.merge(&o.website); - } -} - -impl Crdt for Bucket { - fn merge(&mut self, other: &Self) { - self.state.merge(&other.state); - } -} diff --git a/src/model/prev/v051/mod.rs b/src/model/prev/v051/mod.rs deleted file mode 100644 index 8c1335a5..00000000 --- a/src/model/prev/v051/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod bucket_table; diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs index 7b023d87..57eb7b16 100644 --- a/src/model/s3/block_ref_table.rs +++ b/src/model/s3/block_ref_table.rs @@ -3,8 +3,12 @@ use std::sync::Arc; use garage_db as db; use garage_util::data::*; +use garage_util::error::*; +use garage_util::migrate::Migrate; +use garage_block::CalculateRefcount; use garage_table::crdt::Crdt; +use garage_table::replication::TableShardedReplication; use garage_table::*; use garage_block::manager::*; @@ -84,3 +88,38 @@ impl TableSchema for BlockRefTable { filter.apply(entry.deleted.get()) } } + +pub fn block_ref_recount_fn( + block_ref_table: &Arc>, +) -> CalculateRefcount { + let table = Arc::downgrade(block_ref_table); + Box::new(move |tx: &db::Transaction, block: &Hash| { + let table = table + .upgrade() + .ok_or_message("cannot upgrade weak ptr to block_ref_table") + .map_err(db::TxError::Abort)?; + Ok(calculate_refcount(&table, tx, block)?) + }) +} + +fn calculate_refcount( + block_ref_table: &Table, + tx: &db::Transaction, + block: &Hash, +) -> db::TxResult { + let mut result = 0; + for entry in tx.range(&block_ref_table.data.store, block.as_slice()..)? { + let (key, value) = entry?; + if &key[..32] != block.as_slice() { + break; + } + let value = BlockRef::decode(&value) + .ok_or_message("could not decode block_ref") + .map_err(db::TxError::Abort)?; + assert_eq!(value.block, *block); + if !value.deleted.get() { + result += 1; + } + } + Ok(result) +} diff --git a/src/model/s3/lifecycle_worker.rs b/src/model/s3/lifecycle_worker.rs index 50d4283f..af00437e 100644 --- a/src/model/s3/lifecycle_worker.rs +++ b/src/model/s3/lifecycle_worker.rs @@ -70,7 +70,7 @@ pub fn register_bg_vars( impl LifecycleWorker { pub fn new(garage: Arc, persister: PersisterShared) -> Self { - let today = today(); + let today = today(garage.config.use_local_tz); let last_completed = persister.get_with(|x| { x.last_completed .as_deref() @@ -121,19 +121,13 @@ impl Worker for LifecycleWorker { mpu_aborted, .. } => { - let n_objects = self - .garage - .object_table - .data - .store - .fast_len() - .unwrap_or(None); + let n_objects = self.garage.object_table.data.store.approximate_len().ok(); let progress = match n_objects { - None => "...".to_string(), - Some(total) => format!( + Some(total) if total > 0 => format!( "~{:.2}%", 100. * std::cmp::min(*counter, total) as f32 / total as f32 ), + _ => "...".to_string(), }; WorkerStatus { progress: Some(progress), @@ -211,8 +205,9 @@ impl Worker for LifecycleWorker { async fn wait_for_work(&mut self) -> WorkerState { match &self.state { State::Completed(d) => { + let use_local_tz = self.garage.config.use_local_tz; let next_day = d.succ_opt().expect("no next day"); - let next_start = midnight_ts(next_day); + let next_start = midnight_ts(next_day, use_local_tz); loop { let now = now_msec(); if now < next_start { @@ -224,7 +219,7 @@ impl Worker for LifecycleWorker { break; } } - self.state = State::start(std::cmp::max(next_day, today())); + self.state = State::start(std::cmp::max(next_day, today(use_local_tz))); } State::Running { .. } => (), } @@ -391,20 +386,29 @@ fn check_size_filter(version_data: &ObjectVersionData, filter: &LifecycleFilter) true } -fn midnight_ts(date: NaiveDate) -> u64 { - date.and_hms_opt(0, 0, 0) - .expect("midnight does not exist") - .timestamp_millis() as u64 +fn midnight_ts(date: NaiveDate, use_local_tz: bool) -> u64 { + let midnight = date.and_hms_opt(0, 0, 0).expect("midnight does not exist"); + if use_local_tz { + return midnight + .and_local_timezone(Local) + .single() + .expect("bad local midnight") + .timestamp_millis() as u64; + } + midnight.and_utc().timestamp_millis() as u64 } fn next_date(ts: u64) -> NaiveDate { - NaiveDateTime::from_timestamp_millis(ts as i64) + DateTime::::from_timestamp_millis(ts as i64) .expect("bad timestamp") - .date() + .date_naive() .succ_opt() .expect("no next day") } -fn today() -> NaiveDate { +fn today(use_local_tz: bool) -> NaiveDate { + if use_local_tz { + return Local::now().naive_local().date(); + } Utc::now().naive_utc().date() } diff --git a/src/model/s3/mpu_table.rs b/src/model/s3/mpu_table.rs index 238cbf11..c9f79caf 100644 --- a/src/model/s3/mpu_table.rs +++ b/src/model/s3/mpu_table.rs @@ -17,6 +17,7 @@ pub const PARTS: &str = "parts"; pub const BYTES: &str = "bytes"; mod v09 { + use crate::s3::object_table::ChecksumValue; use garage_util::crdt; use garage_util::data::Uuid; use serde::{Deserialize, Serialize}; @@ -61,6 +62,9 @@ mod v09 { pub version: Uuid, /// ETag of the content of this part (known only once done uploading) pub etag: Option, + /// Checksum requested by x-amz-checksum-algorithm + #[serde(default)] + pub checksum: Option, /// Size of this part (known only once done uploading) pub size: Option, } @@ -155,6 +159,11 @@ impl Crdt for MpuPart { (Some(x), Some(y)) if x < y => other.size, (x, _) => x, }; + self.checksum = match (self.checksum.take(), &other.checksum) { + (None, Some(_)) => other.checksum.clone(), + (Some(x), Some(y)) if x < *y => other.checksum.clone(), + (x, _) => x, + }; } } diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index ebea04bd..6c33b79b 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -17,7 +17,7 @@ pub const OBJECTS: &str = "objects"; pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads"; pub const BYTES: &str = "bytes"; -mod v05 { +mod v08 { use garage_util::data::{Hash, Uuid}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -26,16 +26,16 @@ mod v05 { #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Object { /// The bucket in which the object is stored, used as partition key - pub bucket: String, + pub bucket_id: Uuid, /// The key at which the object is stored in its bucket, used as sorting key pub key: String, - /// The list of currenty stored versions of the object + /// The list of currently stored versions of the object pub(super) versions: Vec, } - /// Informations about a version of an object + /// Information about a version of an object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct ObjectVersion { /// Id of the version @@ -92,45 +92,6 @@ mod v05 { impl garage_util::migrate::InitialFormat for Object {} } -mod v08 { - use garage_util::data::Uuid; - use serde::{Deserialize, Serialize}; - - use super::v05; - - pub use v05::{ - ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta, - ObjectVersionState, - }; - - /// An object - #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct Object { - /// The bucket in which the object is stored, used as partition key - pub bucket_id: Uuid, - - /// The key at which the object is stored in its bucket, used as sorting key - pub key: String, - - /// The list of currenty stored versions of the object - pub(super) versions: Vec, - } - - impl garage_util::migrate::Migrate for Object { - type Previous = v05::Object; - - fn migrate(old: v05::Object) -> Object { - use garage_util::data::blake2sum; - - Object { - bucket_id: blake2sum(old.bucket.as_bytes()), - key: old.key, - versions: old.versions, - } - } - } -} - mod v09 { use garage_util::data::Uuid; use serde::{Deserialize, Serialize}; @@ -148,11 +109,11 @@ mod v09 { /// The key at which the object is stored in its bucket, used as sorting key pub key: String, - /// The list of currenty stored versions of the object + /// The list of currently stored versions of the object pub(super) versions: Vec, } - /// Informations about a version of an object + /// Information about a version of an object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct ObjectVersion { /// Id of the version @@ -210,7 +171,207 @@ mod v09 { } } -pub use v09::*; +mod v010 { + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + + use super::v09; + + /// An object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct Object { + /// The bucket in which the object is stored, used as partition key + pub bucket_id: Uuid, + + /// The key at which the object is stored in its bucket, used as sorting key + pub key: String, + + /// The list of currently stored versions of the object + pub(super) versions: Vec, + } + + /// Information about a version of an object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersion { + /// Id of the version + pub uuid: Uuid, + /// Timestamp of when the object was created + pub timestamp: u64, + /// State of the version + pub state: ObjectVersionState, + } + + /// State of an object version + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionState { + /// The version is being received + Uploading { + /// Indicates whether this is a multipart upload + multipart: bool, + /// Checksum algorithm to use + checksum_algorithm: Option, + /// Encryption params + headers to be included in the final object + encryption: ObjectVersionEncryption, + }, + /// The version is fully received + Complete(ObjectVersionData), + /// The version uploaded containded errors or the upload was explicitly aborted + Aborted, + } + + /// Data stored in object version + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionData { + /// The object was deleted, this Version is a tombstone to mark it as such + DeleteMarker, + /// The object is short, it's stored inlined. + /// It is never compressed. For encrypted objects, it is encrypted using + /// AES256-GCM, like the encrypted headers. + Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec), + /// The object is not short, Hash of first block is stored here, next segments hashes are + /// stored in the version table + FirstBlock(ObjectVersionMeta, Hash), + } + + /// Metadata about the object version + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersionMeta { + /// Size of the object. If object is encrypted/compressed, + /// this is always the size of the unencrypted/uncompressed data + pub size: u64, + /// etag of the object + pub etag: String, + /// Encryption params + headers (encrypted or plaintext) + pub encryption: ObjectVersionEncryption, + } + + /// Encryption information + metadata + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionEncryption { + SseC { + /// Encrypted serialized ObjectVersionInner struct. + /// This is never compressed, just encrypted using AES256-GCM. + #[serde(with = "serde_bytes")] + inner: Vec, + /// Whether data blocks are compressed in addition to being encrypted + /// (compression happens before encryption, whereas for non-encrypted + /// objects, compression is handled at the level of the block manager) + compressed: bool, + }, + Plaintext { + /// Plain-text headers + inner: ObjectVersionMetaInner, + }, + } + + /// Vector of headers, as tuples of the format (header name, header value) + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersionMetaInner { + pub headers: HeaderList, + pub checksum: Option, + } + + pub type HeaderList = Vec<(String, String)>; + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ChecksumAlgorithm { + Crc32, + Crc32c, + Sha1, + Sha256, + } + + /// Checksum value for x-amz-checksum-algorithm + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ChecksumValue { + Crc32(#[serde(with = "serde_bytes")] [u8; 4]), + Crc32c(#[serde(with = "serde_bytes")] [u8; 4]), + Sha1(#[serde(with = "serde_bytes")] [u8; 20]), + Sha256(#[serde(with = "serde_bytes")] [u8; 32]), + } + + impl garage_util::migrate::Migrate for Object { + const VERSION_MARKER: &'static [u8] = b"G010s3ob"; + + type Previous = v09::Object; + + fn migrate(old: v09::Object) -> Object { + Object { + bucket_id: old.bucket_id, + key: old.key, + versions: old.versions.into_iter().map(migrate_version).collect(), + } + } + } + + fn migrate_version(old: v09::ObjectVersion) -> ObjectVersion { + ObjectVersion { + uuid: old.uuid, + timestamp: old.timestamp, + state: match old.state { + v09::ObjectVersionState::Uploading { multipart, headers } => { + ObjectVersionState::Uploading { + multipart, + checksum_algorithm: None, + encryption: migrate_headers(headers), + } + } + v09::ObjectVersionState::Complete(d) => { + ObjectVersionState::Complete(migrate_data(d)) + } + v09::ObjectVersionState::Aborted => ObjectVersionState::Aborted, + }, + } + } + + fn migrate_data(old: v09::ObjectVersionData) -> ObjectVersionData { + match old { + v09::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker, + v09::ObjectVersionData::Inline(meta, data) => { + ObjectVersionData::Inline(migrate_meta(meta), data) + } + v09::ObjectVersionData::FirstBlock(meta, fb) => { + ObjectVersionData::FirstBlock(migrate_meta(meta), fb) + } + } + } + + fn migrate_meta(old: v09::ObjectVersionMeta) -> ObjectVersionMeta { + ObjectVersionMeta { + size: old.size, + etag: old.etag, + encryption: migrate_headers(old.headers), + } + } + + fn migrate_headers(old: v09::ObjectVersionHeaders) -> ObjectVersionEncryption { + use http::header::CONTENT_TYPE; + + let mut new_headers = Vec::with_capacity(old.other.len() + 1); + if old.content_type != "blob" { + new_headers.push((CONTENT_TYPE.as_str().to_string(), old.content_type)); + } + for (name, value) in old.other.into_iter() { + new_headers.push((name, value)); + } + + ObjectVersionEncryption::Plaintext { + inner: ObjectVersionMetaInner { + headers: new_headers, + checksum: None, + }, + } + } + + // Since ObjectVersionMetaInner can now be serialized independently, for the + // purpose of being encrypted, we need it to support migrations on its own + // as well. + impl garage_util::migrate::InitialFormat for ObjectVersionMetaInner { + const VERSION_MARKER: &'static [u8] = b"G010s3om"; + } +} + +pub use v010::*; impl Object { /// Initialize an Object struct from parts @@ -321,6 +482,17 @@ impl Entry for Object { } } +impl ChecksumValue { + pub fn algorithm(&self) -> ChecksumAlgorithm { + match self { + ChecksumValue::Crc32(_) => ChecksumAlgorithm::Crc32, + ChecksumValue::Crc32c(_) => ChecksumAlgorithm::Crc32c, + ChecksumValue::Sha1(_) => ChecksumAlgorithm::Sha1, + ChecksumValue::Sha256(_) => ChecksumAlgorithm::Sha256, + } + } +} + impl Crdt for Object { fn merge(&mut self, other: &Self) { // Merge versions from other into here diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index 5c032f9f..45be5af8 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -11,64 +11,11 @@ use garage_table::*; use crate::s3::block_ref_table::*; -mod v05 { +mod v08 { use garage_util::crdt; use garage_util::data::{Hash, Uuid}; use serde::{Deserialize, Serialize}; - /// A version of an object - #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct Version { - /// UUID of the version, used as partition key - pub uuid: Uuid, - - // Actual data: the blocks for this version - // In the case of a multipart upload, also store the etags - // of individual parts and check them when doing CompleteMultipartUpload - /// Is this version deleted - pub deleted: crdt::Bool, - /// list of blocks of data composing the version - pub blocks: crdt::Map, - /// Etag of each part in case of a multipart upload, empty otherwise - pub parts_etags: crdt::Map, - - // Back link to bucket+key so that we can figure if - // this was deleted later on - /// Bucket in which the related object is stored - pub bucket: String, - /// Key in which the related object is stored - pub key: String, - } - - #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] - pub struct VersionBlockKey { - /// Number of the part - pub part_number: u64, - /// Offset of this sub-segment in its part - pub offset: u64, - } - - /// Informations about a single block - #[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)] - pub struct VersionBlock { - /// Blake2 sum of the block - pub hash: Hash, - /// Size of the block - pub size: u64, - } - - impl garage_util::migrate::InitialFormat for Version {} -} - -mod v08 { - use garage_util::crdt; - use garage_util::data::Uuid; - use serde::{Deserialize, Serialize}; - - use super::v05; - - pub use v05::{VersionBlock, VersionBlockKey}; - /// A version of an object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Version { @@ -93,22 +40,25 @@ mod v08 { pub key: String, } - impl garage_util::migrate::Migrate for Version { - type Previous = v05::Version; - - fn migrate(old: v05::Version) -> Version { - use garage_util::data::blake2sum; - - Version { - uuid: old.uuid, - deleted: old.deleted, - blocks: old.blocks, - parts_etags: old.parts_etags, - bucket_id: blake2sum(old.bucket.as_bytes()), - key: old.key, - } - } + #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct VersionBlockKey { + /// Number of the part + pub part_number: u64, + /// Offset of this sub-segment in its part as sent by the client + /// (before any kind of compression or encryption) + pub offset: u64, } + + /// Information about a single block + #[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct VersionBlock { + /// Blake2 sum of the block + pub hash: Hash, + /// Size of the block, before any kind of compression or encryption + pub size: u64, + } + + impl garage_util::migrate::InitialFormat for Version {} } pub(crate) mod v09 { diff --git a/src/model/snapshot.rs b/src/model/snapshot.rs new file mode 100644 index 00000000..8e8995f9 --- /dev/null +++ b/src/model/snapshot.rs @@ -0,0 +1,142 @@ +use std::fs; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use rand::prelude::*; +use tokio::sync::watch; + +use garage_util::background::*; +use garage_util::error::*; + +use crate::garage::Garage; + +// The two most recent snapshots are kept +const KEEP_SNAPSHOTS: usize = 2; + +static SNAPSHOT_MUTEX: Mutex<()> = Mutex::new(()); + +// ================ snapshotting logic ===================== + +/// Run snapshot_metadata in a blocking thread and async await on it +pub async fn async_snapshot_metadata(garage: &Arc) -> Result<(), Error> { + let garage = garage.clone(); + let worker = tokio::task::spawn_blocking(move || snapshot_metadata(&garage)); + worker.await.unwrap()?; + Ok(()) +} + +/// Take a snapshot of the metadata database, and erase older +/// snapshots if necessary. +/// This is not an async function, it should be spawned on a thread pool +pub fn snapshot_metadata(garage: &Garage) -> Result<(), Error> { + let lock = match SNAPSHOT_MUTEX.try_lock() { + Ok(lock) => lock, + Err(_) => { + return Err(Error::Message( + "Cannot acquire lock, another snapshot might be in progress".into(), + )) + } + }; + + let snapshots_dir = match &garage.config.metadata_snapshots_dir { + Some(d) => d.clone(), + None => { + let mut default_snapshots_dir = garage.config.metadata_dir.clone(); + default_snapshots_dir.push("snapshots"); + default_snapshots_dir + } + }; + fs::create_dir_all(&snapshots_dir)?; + + let mut new_path = snapshots_dir.clone(); + new_path.push(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)); + + info!("Snapshotting metadata db to {}", new_path.display()); + garage.db.snapshot(&new_path)?; + info!("Metadata db snapshot finished"); + + if let Err(e) = cleanup_snapshots(&snapshots_dir) { + error!("Failed to do cleanup in snapshots directory: {}", e); + } + + drop(lock); + + Ok(()) +} + +fn cleanup_snapshots(snapshots_dir: &PathBuf) -> Result<(), Error> { + let mut snapshots = + fs::read_dir(&snapshots_dir)?.collect::, std::io::Error>>()?; + + snapshots.retain(|x| x.file_name().len() > 8); + snapshots.sort_by_key(|x| x.file_name()); + + for to_delete in snapshots.iter().rev().skip(KEEP_SNAPSHOTS) { + let path = snapshots_dir.join(to_delete.path()); + if to_delete.metadata()?.file_type().is_dir() { + for file in fs::read_dir(&path)? { + let file = file?; + if file.metadata()?.is_file() { + fs::remove_file(path.join(file.path()))?; + } + } + std::fs::remove_dir(&path)?; + } else { + std::fs::remove_file(&path)?; + } + } + Ok(()) +} + +// ================ auto snapshot worker ===================== + +pub struct AutoSnapshotWorker { + garage: Arc, + next_snapshot: Instant, + snapshot_interval: Duration, +} + +impl AutoSnapshotWorker { + pub(crate) fn new(garage: Arc, snapshot_interval: Duration) -> Self { + Self { + garage, + snapshot_interval, + next_snapshot: Instant::now() + (snapshot_interval / 2), + } + } +} + +#[async_trait] +impl Worker for AutoSnapshotWorker { + fn name(&self) -> String { + "Metadata snapshot worker".into() + } + fn status(&self) -> WorkerStatus { + WorkerStatus { + freeform: vec![format!( + "Next snapshot: {}", + (chrono::Utc::now() + (self.next_snapshot - Instant::now())).to_rfc3339() + )], + ..Default::default() + } + } + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + if Instant::now() < self.next_snapshot { + return Ok(WorkerState::Idle); + } + + async_snapshot_metadata(&self.garage).await?; + + let rand_factor = 1f32 + thread_rng().gen::() / 5f32; + self.next_snapshot = Instant::now() + self.snapshot_interval.mul_f32(rand_factor); + + Ok(WorkerState::Idle) + } + async fn wait_for_work(&mut self) -> WorkerState { + tokio::time::sleep_until(self.next_snapshot.into()).await; + WorkerState::Busy + } +} diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml new file mode 100644 index 00000000..71f42c68 --- /dev/null +++ b/src/net/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "garage_net" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Networking library for Garage RPC communication, forked from Netapp" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +[features] +default = [] +telemetry = ["opentelemetry", "opentelemetry-contrib"] + +[dependencies] +futures.workspace = true +pin-project.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tokio-stream.workspace = true + +serde.workspace = true +rmp-serde.workspace = true +hex.workspace = true + +rand.workspace = true + +log.workspace = true +arc-swap.workspace = true +thiserror.workspace = true +bytes.workspace = true +cfg-if.workspace = true + +sodiumoxide.workspace = true +kuska-handshake.workspace = true + +opentelemetry = { workspace = true, optional = true } +opentelemetry-contrib = { workspace = true, optional = true } + +[dev-dependencies] +pretty_env_logger.workspace = true diff --git a/src/net/bytes_buf.rs b/src/net/bytes_buf.rs new file mode 100644 index 00000000..1d928ffb --- /dev/null +++ b/src/net/bytes_buf.rs @@ -0,0 +1,199 @@ +use std::cmp::Ordering; +use std::collections::VecDeque; + +use bytes::BytesMut; + +use crate::stream::ByteStream; + +pub use bytes::Bytes; + +/// A circular buffer of bytes, internally represented as a list of Bytes +/// for optimization, but that for all intent and purposes acts just like +/// a big byte slice which can be extended on the right and from which +/// stuff can be taken on the left. +pub struct BytesBuf { + buf: VecDeque, + buf_len: usize, +} + +impl BytesBuf { + /// Creates a new empty BytesBuf + pub fn new() -> Self { + Self { + buf: VecDeque::new(), + buf_len: 0, + } + } + + /// Returns the number of bytes stored in the BytesBuf + #[inline] + pub fn len(&self) -> usize { + self.buf_len + } + + /// Returns true iff the BytesBuf contains zero bytes + #[inline] + pub fn is_empty(&self) -> bool { + self.buf_len == 0 + } + + /// Adds some bytes to the right of the buffer + pub fn extend(&mut self, b: Bytes) { + if !b.is_empty() { + self.buf_len += b.len(); + self.buf.push_back(b); + } + } + + /// Takes the whole content of the buffer and returns it as a single Bytes unit + pub fn take_all(&mut self) -> Bytes { + if self.buf.is_empty() { + Bytes::new() + } else if self.buf.len() == 1 { + self.buf_len = 0; + self.buf.pop_back().unwrap() + } else { + let mut ret = BytesMut::with_capacity(self.buf_len); + for b in self.buf.iter() { + ret.extend_from_slice(&b[..]); + } + self.buf.clear(); + self.buf_len = 0; + ret.freeze() + } + } + + /// Takes at most max_len bytes from the left of the buffer + pub fn take_max(&mut self, max_len: usize) -> Bytes { + if self.buf_len <= max_len { + self.take_all() + } else { + self.take_exact_ok(max_len) + } + } + + /// Take exactly len bytes from the left of the buffer, returns None if + /// the BytesBuf doesn't contain enough data + pub fn take_exact(&mut self, len: usize) -> Option { + if self.buf_len < len { + None + } else { + Some(self.take_exact_ok(len)) + } + } + + fn take_exact_ok(&mut self, len: usize) -> Bytes { + assert!(len <= self.buf_len); + let front = self.buf.pop_front().unwrap(); + match front.len().cmp(&len) { + Ordering::Greater => { + self.buf.push_front(front.slice(len..)); + self.buf_len -= len; + front.slice(..len) + } + Ordering::Equal => { + self.buf_len -= len; + front + } + Ordering::Less => { + let mut ret = BytesMut::with_capacity(len); + ret.extend_from_slice(&front[..]); + self.buf_len -= front.len(); + while ret.len() < len { + let front = self.buf.pop_front().unwrap(); + if front.len() > len - ret.len() { + let take = len - ret.len(); + ret.extend_from_slice(&front[..take]); + self.buf.push_front(front.slice(take..)); + self.buf_len -= take; + break; + } else { + ret.extend_from_slice(&front[..]); + self.buf_len -= front.len(); + } + } + ret.freeze() + } + } + } + + /// Return the internal sequence of Bytes slices that make up the buffer + pub fn into_slices(self) -> VecDeque { + self.buf + } + + /// Return the entire buffer concatenated into a single big Bytes + pub fn into_bytes(mut self) -> Bytes { + self.take_all() + } + + /// Return the content as a stream of individual chunks + pub fn into_stream(self) -> ByteStream { + use futures::stream::StreamExt; + Box::pin(futures::stream::iter(self.buf).map(|x| Ok(x))) + } +} + +impl Default for BytesBuf { + fn default() -> Self { + Self::new() + } +} + +impl From for BytesBuf { + fn from(b: Bytes) -> BytesBuf { + let mut ret = BytesBuf::new(); + ret.extend(b); + ret + } +} + +impl From for Bytes { + fn from(mut b: BytesBuf) -> Bytes { + b.take_all() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bytes_buf() { + let mut buf = BytesBuf::new(); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + + buf.extend(Bytes::from(b"Hello, world!".to_vec())); + assert!(buf.len() == 13); + assert!(!buf.is_empty()); + + buf.extend(Bytes::from(b"1234567890".to_vec())); + assert!(buf.len() == 23); + assert!(!buf.is_empty()); + + assert_eq!( + buf.take_all(), + Bytes::from(b"Hello, world!1234567890".to_vec()) + ); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + + buf.extend(Bytes::from(b"1234567890".to_vec())); + buf.extend(Bytes::from(b"Hello, world!".to_vec())); + assert!(buf.len() == 23); + assert!(!buf.is_empty()); + + assert_eq!(buf.take_max(12), Bytes::from(b"1234567890He".to_vec())); + assert!(buf.len() == 11); + + assert_eq!(buf.take_exact(12), None); + assert!(buf.len() == 11); + assert_eq!( + buf.take_exact(11), + Some(Bytes::from(b"llo, world!".to_vec())) + ); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + } +} diff --git a/src/net/client.rs b/src/net/client.rs new file mode 100644 index 00000000..20e1dacd --- /dev/null +++ b/src/net/client.rs @@ -0,0 +1,290 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::atomic::{self, AtomicU32}; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + +use arc_swap::ArcSwapOption; +use bytes::Bytes; +use log::{debug, error, trace}; + +use futures::io::AsyncReadExt; +use futures::Stream; +use kuska_handshake::async_std::{handshake_client, BoxStream}; +use tokio::net::TcpStream; +use tokio::select; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio_util::compat::*; + +#[cfg(feature = "telemetry")] +use opentelemetry::{ + trace::{FutureExt, Span, SpanKind, TraceContextExt, Tracer}, + Context, KeyValue, +}; +#[cfg(feature = "telemetry")] +use opentelemetry_contrib::trace::propagator::binary::*; + +use crate::error::*; +use crate::message::*; +use crate::netapp::*; +use crate::recv::*; +use crate::send::*; +use crate::stream::*; +use crate::util::*; + +pub(crate) struct ClientConn { + pub(crate) remote_addr: SocketAddr, + pub(crate) peer_id: NodeID, + + query_send: ArcSwapOption>, + + next_query_number: AtomicU32, + inflight: Mutex>>, +} + +impl ClientConn { + pub(crate) async fn init( + netapp: Arc, + socket: TcpStream, + peer_id: NodeID, + ) -> Result<(), Error> { + let remote_addr = socket.peer_addr()?; + let mut socket = socket.compat(); + + // Do handshake to authenticate and prove our identity to server + let handshake = handshake_client( + &mut socket, + netapp.netid.clone(), + netapp.id, + netapp.privkey.clone(), + peer_id, + ) + .await?; + + debug!( + "Handshake complete (client) with {}@{}", + hex::encode(peer_id), + remote_addr + ); + + // Create BoxStream layer that encodes content + let (read, write) = socket.split(); + let (mut read, write) = + BoxStream::from_handshake(read, write, handshake, 0x8000).split_read_write(); + + // Before doing anything, receive version tag and + // check they are running the same version as us + let mut their_version_tag = VersionTag::default(); + read.read_exact(&mut their_version_tag[..]).await?; + if their_version_tag != netapp.version_tag { + let msg = format!( + "different version tags: {} (theirs) vs. {} (ours)", + hex::encode(their_version_tag), + hex::encode(netapp.version_tag) + ); + error!("Cannot connect to {}: {}", hex::encode(&peer_id[..8]), msg); + return Err(Error::VersionMismatch(msg)); + } + + // Build and launch stuff that manages sending requests client-side + let (query_send, query_recv) = mpsc::unbounded_channel(); + + let (stop_recv_loop, stop_recv_loop_recv) = watch::channel(false); + + let conn = Arc::new(ClientConn { + remote_addr, + peer_id, + next_query_number: AtomicU32::from(RequestID::default()), + query_send: ArcSwapOption::new(Some(Arc::new(query_send))), + inflight: Mutex::new(HashMap::new()), + }); + + netapp.connected_as_client(peer_id, conn.clone()); + + let debug_name = format!("CLI {}", hex::encode(&peer_id[..8])); + + tokio::spawn(async move { + let debug_name_2 = debug_name.clone(); + let send_future = tokio::spawn(conn.clone().send_loop(query_recv, write, debug_name_2)); + + let conn2 = conn.clone(); + let recv_future = tokio::spawn(async move { + select! { + r = conn2.recv_loop(read, debug_name) => r, + _ = await_exit(stop_recv_loop_recv) => Ok(()) + } + }); + + send_future.await.log_err("ClientConn send_loop"); + + // FIXME: should do here: wait for inflight requests to all have their response + stop_recv_loop + .send(true) + .log_err("ClientConn send true to stop_recv_loop"); + + recv_future.await.log_err("ClientConn recv_loop"); + + // Make sure we don't wait on any more requests that won't + // have a response + conn.inflight.lock().unwrap().clear(); + + netapp.disconnected_as_client(&peer_id, conn); + }); + + Ok(()) + } + + pub fn close(&self) { + self.query_send.store(None); + } + + pub(crate) async fn call( + self: Arc, + req: Req, + path: &str, + prio: RequestPriority, + ) -> Result, Error> + where + T: Message, + { + let query_send = self.query_send.load_full().ok_or(Error::ConnectionClosed)?; + + let id = self + .next_query_number + .fetch_add(1, atomic::Ordering::Relaxed); + + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let tracer = opentelemetry::global::tracer("netapp"); + let mut span = tracer.span_builder(format!("RPC >> {}", path)) + .with_kind(SpanKind::Client) + .start(&tracer); + let propagator = BinaryPropagator::new(); + let telemetry_id: Bytes = propagator.to_bytes(span.span_context()).to_vec().into(); + } else { + let telemetry_id: Bytes = Bytes::new(); + } + }; + + // Encode request + let req_enc = req.into_enc(prio, path.as_bytes().to_vec().into(), telemetry_id); + let req_msg_len = req_enc.msg.len(); + let (req_stream, req_order) = req_enc.encode(); + + // Send request through + let (resp_send, resp_recv) = oneshot::channel(); + let old = self.inflight.lock().unwrap().insert(id, resp_send); + if let Some(old_ch) = old { + error!( + "Too many inflight requests! RequestID collision. Interrupting previous request." + ); + let _ = old_ch.send(Box::pin(futures::stream::once(async move { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "RequestID collision, too many inflight requests", + )) + }))); + } + + debug!( + "request: query_send {}, path {}, prio {} (serialized message: {} bytes)", + id, path, prio, req_msg_len + ); + + #[cfg(feature = "telemetry")] + span.set_attribute(KeyValue::new("len_query_msg", req_msg_len as i64)); + + query_send.send(SendItem::Stream(id, prio, req_order, req_stream))?; + + let canceller = CancelOnDrop::new(id, query_send.as_ref().clone()); + + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let stream = resp_recv + .with_context(Context::current_with_span(span)) + .await?; + } else { + let stream = resp_recv.await?; + } + } + + let stream = Box::pin(canceller.for_stream(stream)); + + let resp_enc = RespEnc::decode(stream).await?; + debug!("client: got response to request {} (path {})", id, path); + Resp::from_enc(resp_enc) + } +} + +impl SendLoop for ClientConn {} + +impl RecvLoop for ClientConn { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream) { + trace!("ClientConn recv_handler {}", id); + + let mut inflight = self.inflight.lock().unwrap(); + if let Some(ch) = inflight.remove(&id) { + if ch.send(stream).is_err() { + debug!("Could not send request response, probably because request was interrupted. Dropping response."); + } + } else { + debug!("Got unexpected response to request {}, dropping it", id); + } + } +} + +// ---- + +struct CancelOnDrop { + id: RequestID, + query_send: mpsc::UnboundedSender, +} + +impl CancelOnDrop { + fn new(id: RequestID, query_send: mpsc::UnboundedSender) -> Self { + Self { id, query_send } + } + fn for_stream(self, stream: ByteStream) -> CancelOnDropStream { + CancelOnDropStream { + cancel: Some(self), + stream, + } + } +} + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + trace!("cancelling request {}", self.id); + let _ = self.query_send.send(SendItem::Cancel(self.id)); + } +} + +#[pin_project::pin_project] +struct CancelOnDropStream { + cancel: Option, + #[pin] + stream: ByteStream, +} + +impl Stream for CancelOnDropStream { + type Item = Packet; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + let this = self.project(); + let res = this.stream.poll_next(cx); + if matches!(res, Poll::Ready(None)) { + if let Some(c) = this.cancel.take() { + std::mem::forget(c) + } + } + res + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} diff --git a/src/net/endpoint.rs b/src/net/endpoint.rs new file mode 100644 index 00000000..3ab1048a --- /dev/null +++ b/src/net/endpoint.rs @@ -0,0 +1,199 @@ +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use arc_swap::ArcSwapOption; +use futures::future::{BoxFuture, FutureExt}; + +use crate::error::Error; +use crate::message::*; +use crate::netapp::*; + +/// This trait should be implemented by an object of your application +/// that can handle a message of type `M`, if it wishes to handle +/// streams attached to the request and/or to send back streams +/// attached to the response.. +/// +/// The handler object should be in an Arc, see `Endpoint::set_handler` +pub trait StreamingEndpointHandler: Send + Sync +where + M: Message, +{ + fn handle(self: &Arc, m: Req, from: NodeID) -> impl Future> + Send; +} + +/// If one simply wants to use an endpoint in a client fashion, +/// without locally serving requests to that endpoint, +/// use the unit type `()` as the handler type: +/// it will panic if it is ever made to handle request. +impl EndpointHandler for () { + async fn handle(self: &Arc<()>, _m: &M, _from: NodeID) -> M::Response { + panic!("This endpoint should not have a local handler."); + } +} + +// ---- + +/// This trait should be implemented by an object of your application +/// that can handle a message of type `M`, in the cases where it doesn't +/// care about attached stream in the request nor in the response. +pub trait EndpointHandler: Send + Sync +where + M: Message, +{ + fn handle(self: &Arc, m: &M, from: NodeID) -> impl Future + Send; +} + +impl StreamingEndpointHandler for T +where + T: EndpointHandler, + M: Message, +{ + async fn handle(self: &Arc, mut m: Req, from: NodeID) -> Resp { + // Immediately drop stream to ignore all data that comes in, + // instead of buffering it indefinitely + drop(m.take_stream()); + Resp::new(EndpointHandler::handle(self, m.msg(), from).await) + } +} + +// ---- + +/// This struct represents an endpoint for message of type `M`. +/// +/// Creating a new endpoint is done by calling `NetApp::endpoint`. +/// An endpoint is identified primarily by its path, which is specified +/// at creation time. +/// +/// An `Endpoint` is used both to send requests to remote nodes, +/// and to specify the handler for such requests on the local node. +/// The type `H` represents the type of the handler object for +/// endpoint messages (see `StreamingEndpointHandler`). +pub struct Endpoint +where + M: Message, + H: StreamingEndpointHandler, +{ + _phantom: PhantomData, + netapp: Arc, + path: String, + handler: ArcSwapOption, +} + +impl Endpoint +where + M: Message, + H: StreamingEndpointHandler, +{ + pub(crate) fn new(netapp: Arc, path: String) -> Self { + Self { + _phantom: PhantomData::default(), + netapp, + path, + handler: ArcSwapOption::from(None), + } + } + + /// Get the path of this endpoint + pub fn path(&self) -> &str { + &self.path + } + + /// Set the object that is responsible of handling requests to + /// this endpoint on the local node. + pub fn set_handler(&self, h: Arc) { + self.handler.swap(Some(h)); + } + + /// Call this endpoint on a remote node (or on the local node, + /// for that matter). This function invokes the full version that + /// allows to attach a stream to the request and to + /// receive such a stream attached to the response. + pub async fn call_streaming( + &self, + target: &NodeID, + req: T, + prio: RequestPriority, + ) -> Result, Error> + where + T: IntoReq, + { + if *target == self.netapp.id { + match self.handler.load_full() { + None => Err(Error::NoHandler), + Some(h) => Ok(h.handle(req.into_req_local(), self.netapp.id).await), + } + } else { + let conn = self + .netapp + .client_conns + .read() + .unwrap() + .get(target) + .cloned(); + match conn { + None => Err(Error::Message(format!( + "Not connected: {}", + hex::encode(&target[..8]) + ))), + Some(c) => c.call(req.into_req()?, self.path.as_str(), prio).await, + } + } + } + + /// Call this endpoint on a remote node. This function is the simplified + /// version that doesn't allow to have streams attached to the request + /// or the response; see `call_streaming` for the full version. + pub async fn call( + &self, + target: &NodeID, + req: M, + prio: RequestPriority, + ) -> Result<::Response, Error> { + Ok(self.call_streaming(target, req, prio).await?.into_msg()) + } +} + +// ---- Internal stuff ---- + +pub(crate) type DynEndpoint = Box; + +pub(crate) trait GenericEndpoint { + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result>; + fn drop_handler(&self); + fn clone_endpoint(&self) -> DynEndpoint; +} + +#[derive(Clone)] +pub(crate) struct EndpointArc(pub(crate) Arc>) +where + M: Message, + H: StreamingEndpointHandler; + +impl GenericEndpoint for EndpointArc +where + M: Message, + H: StreamingEndpointHandler + 'static, +{ + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result> { + async move { + match self.0.handler.load_full() { + None => Err(Error::NoHandler), + Some(h) => { + let req = Req::from_enc(req_enc)?; + let res = h.handle(req, from).await; + Ok(res.into_enc()?) + } + } + } + .boxed() + } + + fn drop_handler(&self) { + self.0.handler.swap(None); + } + + fn clone_endpoint(&self) -> DynEndpoint { + Box::new(Self(self.0.clone())) + } +} diff --git a/src/net/error.rs b/src/net/error.rs new file mode 100644 index 00000000..f67794ed --- /dev/null +++ b/src/net/error.rs @@ -0,0 +1,126 @@ +use std::io; + +use log::error; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("IO error: {0}")] + Io(#[from] io::Error), + + #[error("Messagepack encode error: {0}")] + RMPEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RMPDecode(#[from] rmp_serde::decode::Error), + + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), + + #[error("oneshot receive error: {0}")] + OneshotRecv(#[from] tokio::sync::oneshot::error::RecvError), + + #[error("Handshake error: {0}")] + Handshake(#[from] kuska_handshake::async_std::Error), + + #[error("UTF8 error: {0}")] + UTF8(#[from] std::string::FromUtf8Error), + + #[error("Framing protocol error")] + Framing, + + #[error("Remote error ({0:?}): {1}")] + Remote(io::ErrorKind, String), + + #[error("Request ID collision")] + IdCollision, + + #[error("{0}")] + Message(String), + + #[error("No handler / shutting down")] + NoHandler, + + #[error("Connection closed")] + ConnectionClosed, + + #[error("Version mismatch: {0}")] + VersionMismatch(String), +} + +impl From> for Error { + fn from(_e: tokio::sync::watch::error::SendError) -> Error { + Error::Message("Watch send error".into()) + } +} + +impl From> for Error { + fn from(_e: tokio::sync::mpsc::error::SendError) -> Error { + Error::Message("MPSC send error".into()) + } +} + +/// The trait adds a `.log_err()` method on `Result<(), E>` types, +/// which dismisses the error by logging it to stderr. +pub trait LogError { + fn log_err(self, msg: &'static str); +} + +impl LogError for Result<(), E> +where + E: Into, +{ + fn log_err(self, msg: &'static str) { + if let Err(e) = self { + error!("Error: {}: {}", msg, Into::::into(e)); + }; + } +} + +impl LogError for Result +where + T: LogError, + E: Into, +{ + fn log_err(self, msg: &'static str) { + match self { + Err(e) => error!("Error: {}: {}", msg, Into::::into(e)), + Ok(x) => x.log_err(msg), + } + } +} + +// ---- Helpers for serializing I/O Errors + +pub(crate) fn u8_to_io_errorkind(v: u8) -> std::io::ErrorKind { + use std::io::ErrorKind; + match v { + 101 => ErrorKind::ConnectionAborted, + 102 => ErrorKind::BrokenPipe, + 103 => ErrorKind::WouldBlock, + 104 => ErrorKind::InvalidInput, + 105 => ErrorKind::InvalidData, + 106 => ErrorKind::TimedOut, + 107 => ErrorKind::Interrupted, + 108 => ErrorKind::UnexpectedEof, + 109 => ErrorKind::OutOfMemory, + 110 => ErrorKind::ConnectionReset, + _ => ErrorKind::Other, + } +} + +pub(crate) fn io_errorkind_to_u8(kind: std::io::ErrorKind) -> u8 { + use std::io::ErrorKind; + match kind { + ErrorKind::ConnectionAborted => 101, + ErrorKind::BrokenPipe => 102, + ErrorKind::WouldBlock => 103, + ErrorKind::InvalidInput => 104, + ErrorKind::InvalidData => 105, + ErrorKind::TimedOut => 106, + ErrorKind::Interrupted => 107, + ErrorKind::UnexpectedEof => 108, + ErrorKind::OutOfMemory => 109, + ErrorKind::ConnectionReset => 110, + _ => 100, + } +} diff --git a/src/net/lib.rs b/src/net/lib.rs new file mode 100644 index 00000000..8e30e40f --- /dev/null +++ b/src/net/lib.rs @@ -0,0 +1,35 @@ +//! Netapp is a Rust library that takes care of a few common tasks in distributed software: +//! +//! - establishing secure connections +//! - managing connection lifetime, reconnecting on failure +//! - checking peer's state +//! - peer discovery +//! - query/response message passing model for communications +//! - multiplexing transfers over a connection +//! - overlay networks: full mesh, and soon other methods +//! +//! Of particular interest, read the documentation for the `netapp::NetApp` type, +//! the `message::Message` trait, and `proto::RequestPriority` to learn more +//! about message priorization. +//! Also check out the examples to learn how to use this crate. + +pub mod bytes_buf; +pub mod error; +pub mod stream; +pub mod util; + +pub mod endpoint; +pub mod message; + +mod client; +mod recv; +mod send; +mod server; + +pub mod netapp; +pub mod peering; + +pub use crate::netapp::*; + +#[cfg(test)] +mod test; diff --git a/src/net/message.rs b/src/net/message.rs new file mode 100644 index 00000000..59afb058 --- /dev/null +++ b/src/net/message.rs @@ -0,0 +1,540 @@ +use std::fmt; +use std::marker::PhantomData; +use std::sync::Arc; + +use bytes::{BufMut, Bytes, BytesMut}; +use rand::prelude::*; +use serde::{Deserialize, Serialize}; + +use futures::stream::StreamExt; + +use crate::error::*; +use crate::stream::*; +use crate::util::*; + +/// Priority of a request (click to read more about priorities). +/// +/// This priority value is used to priorize messages +/// in the send queue of the client, and their responses in the send queue of the +/// server. Lower values mean higher priority. +/// +/// This mechanism is useful for messages bigger than the maximum chunk size +/// (set at `0x4000` bytes), such as large file transfers. +/// In such case, all of the messages in the send queue with the highest priority +/// will take turns to send individual chunks, in a round-robin fashion. +/// Once all highest priority messages are sent successfully, the messages with +/// the next highest priority will begin being sent in the same way. +/// +/// The same priority value is given to a request and to its associated response. +pub type RequestPriority = u8; + +// Usage of priority levels in Garage: +// +// PRIO_HIGH +// for liveness check events such as pings and important +// reconfiguration events such as layout changes +// +// PRIO_NORMAL +// for standard interactive requests to exchange metadata +// +// PRIO_NORMAL | PRIO_SECONDARY +// for standard interactive requests to exchange block data +// +// PRIO_BACKGROUND +// for background resync requests to exchange metadata +// PRIO_BACKGROUND | PRIO_SECONDARY +// for background resync requests to exchange block data + +/// Priority class: high +pub const PRIO_HIGH: RequestPriority = 0x20; +/// Priority class: normal +pub const PRIO_NORMAL: RequestPriority = 0x40; +/// Priority class: background +pub const PRIO_BACKGROUND: RequestPriority = 0x80; + +/// Priority: primary among given class +pub const PRIO_PRIMARY: RequestPriority = 0x00; +/// Priority: secondary among given class (ex: `PRIO_HIGH | PRIO_SECONDARY`) +pub const PRIO_SECONDARY: RequestPriority = 0x01; + +// ---- + +/// An order tag can be added to a message or a response to indicate +/// whether it should be sent after or before other messages with order tags +/// referencing a same stream +#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +pub struct OrderTag(pub(crate) u64, pub(crate) u64); + +/// A stream is an opaque identifier that defines a set of messages +/// or responses that are ordered wrt one another using to order tags. +#[derive(Clone, Copy)] +pub struct OrderTagStream(u64); + +impl OrderTag { + /// Create a new stream from which to generate order tags. Example: + /// ```ignore + /// let stream = OrderTag.stream(); + /// let tag_1 = stream.order(1); + /// let tag_2 = stream.order(2); + /// ``` + pub fn stream() -> OrderTagStream { + OrderTagStream(thread_rng().gen()) + } +} +impl OrderTagStream { + /// Create the order tag for message `order` in this stream + pub fn order(&self, order: u64) -> OrderTag { + OrderTag(self.0, order) + } +} + +// ---- + +/// This trait should be implemented by all messages your application +/// wants to handle. It specifies which data type should be sent +/// as a response to this message in the RPC protocol. +pub trait Message: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static { + /// The type of the response that is sent in response to this message + type Response: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static; +} + +// ---- + +/// The Req is a helper object used to create requests and attach them +/// a stream of data. If the stream is a fixed Bytes and not a ByteStream, +/// Req is cheaply cloneable to allow the request to be sent to different +/// peers (Clone will panic if the stream is a ByteStream). +pub struct Req { + pub(crate) msg: Arc, + pub(crate) msg_ser: Option, + pub(crate) stream: AttachedStream, + pub(crate) order_tag: Option, +} + +impl Req { + /// Creates a new request from a base message `M` + pub fn new(v: M) -> Result { + Ok(v.into_req()?) + } + + /// Attach a stream to message in request, where the stream is streamed + /// from a fixed `Bytes` buffer + pub fn with_stream_from_buffer(self, b: Bytes) -> Self { + Self { + stream: AttachedStream::Fixed(b), + ..self + } + } + + /// Attach a stream to message in request, where the stream is + /// an instance of `ByteStream`. Note than when a `Req` has an attached + /// stream which is a `ByteStream` instance, it can no longer be cloned + /// to be sent to different nodes (`.clone()` will panic) + pub fn with_stream(self, b: ByteStream) -> Self { + Self { + stream: AttachedStream::Stream(b), + ..self + } + } + + /// Add an order tag to this request to indicate in which order it should + /// be sent. + pub fn with_order_tag(self, order_tag: OrderTag) -> Self { + Self { + order_tag: Some(order_tag), + ..self + } + } + + /// Get a reference to the message `M` contained in this request + pub fn msg(&self) -> &M { + &self.msg + } + + /// Takes out the stream attached to this request, if any + pub fn take_stream(&mut self) -> Option { + std::mem::replace(&mut self.stream, AttachedStream::None).into_stream() + } + + pub(crate) fn into_enc( + self, + prio: RequestPriority, + path: Bytes, + telemetry_id: Bytes, + ) -> ReqEnc { + ReqEnc { + prio, + path, + telemetry_id, + msg: self.msg_ser.unwrap(), + stream: self.stream.into_stream(), + order_tag: self.order_tag, + } + } + + pub(crate) fn from_enc(enc: ReqEnc) -> Result { + let msg = rmp_serde::decode::from_slice(&enc.msg)?; + Ok(Req { + msg: Arc::new(msg), + msg_ser: Some(enc.msg), + stream: enc + .stream + .map(AttachedStream::Stream) + .unwrap_or(AttachedStream::None), + order_tag: enc.order_tag, + }) + } +} + +/// `IntoReq` represents any object that can be transformed into `Req` +pub trait IntoReq { + /// Transform the object into a `Req`, serializing the message M + /// to be sent to remote nodes + fn into_req(self) -> Result, rmp_serde::encode::Error>; + /// Transform the object into a `Req`, skipping the serialization + /// of message M, in the case we are not sending this RPC message to + /// a remote node + fn into_req_local(self) -> Req; +} + +impl IntoReq for M { + fn into_req(self) -> Result, rmp_serde::encode::Error> { + let msg_ser = rmp_to_vec_all_named(&self)?; + Ok(Req { + msg: Arc::new(self), + msg_ser: Some(Bytes::from(msg_ser)), + stream: AttachedStream::None, + order_tag: None, + }) + } + fn into_req_local(self) -> Req { + Req { + msg: Arc::new(self), + msg_ser: None, + stream: AttachedStream::None, + order_tag: None, + } + } +} + +impl IntoReq for Req { + fn into_req(self) -> Result, rmp_serde::encode::Error> { + Ok(self) + } + fn into_req_local(self) -> Req { + self + } +} + +impl Clone for Req { + fn clone(&self) -> Self { + let stream = match &self.stream { + AttachedStream::None => AttachedStream::None, + AttachedStream::Fixed(b) => AttachedStream::Fixed(b.clone()), + AttachedStream::Stream(_) => { + panic!("Cannot clone a Req<_> with a non-buffer attached stream") + } + }; + Self { + msg: self.msg.clone(), + msg_ser: self.msg_ser.clone(), + stream, + order_tag: self.order_tag, + } + } +} + +impl fmt::Debug for Req +where + M: Message + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "Req[{:?}", self.msg)?; + match &self.stream { + AttachedStream::None => write!(f, "]"), + AttachedStream::Fixed(b) => write!(f, "; stream=buf:{}]", b.len()), + AttachedStream::Stream(_) => write!(f, "; stream]"), + } + } +} + +// ---- + +/// The Resp represents a full response from a RPC that may have +/// an attached stream. +pub struct Resp { + pub(crate) _phantom: PhantomData, + pub(crate) msg: M::Response, + pub(crate) stream: AttachedStream, + pub(crate) order_tag: Option, +} + +impl Resp { + /// Creates a new response from a base response message + pub fn new(v: M::Response) -> Self { + Resp { + _phantom: Default::default(), + msg: v, + stream: AttachedStream::None, + order_tag: None, + } + } + + /// Attach a stream to message in response, where the stream is streamed + /// from a fixed `Bytes` buffer + pub fn with_stream_from_buffer(self, b: Bytes) -> Self { + Self { + stream: AttachedStream::Fixed(b), + ..self + } + } + + /// Attach a stream to message in response, where the stream is + /// an instance of `ByteStream`. + pub fn with_stream(self, b: ByteStream) -> Self { + Self { + stream: AttachedStream::Stream(b), + ..self + } + } + + /// Add an order tag to this response to indicate in which order it should + /// be sent. + pub fn with_order_tag(self, order_tag: OrderTag) -> Self { + Self { + order_tag: Some(order_tag), + ..self + } + } + + /// Get a reference to the response message contained in this request + pub fn msg(&self) -> &M::Response { + &self.msg + } + + /// Transforms the `Resp` into the response message it contains, + /// dropping everything else (including attached data stream) + pub fn into_msg(self) -> M::Response { + self.msg + } + + /// Transforms the `Resp` into, on the one side, the response message + /// it contains, and on the other side, the associated data stream + /// if it exists + pub fn into_parts(self) -> (M::Response, Option) { + (self.msg, self.stream.into_stream()) + } + + pub(crate) fn into_enc(self) -> Result { + Ok(RespEnc { + msg: rmp_to_vec_all_named(&self.msg)?.into(), + stream: self.stream.into_stream(), + order_tag: self.order_tag, + }) + } + + pub(crate) fn from_enc(enc: RespEnc) -> Result { + let msg = rmp_serde::decode::from_slice(&enc.msg)?; + Ok(Self { + _phantom: Default::default(), + msg, + stream: enc + .stream + .map(AttachedStream::Stream) + .unwrap_or(AttachedStream::None), + order_tag: enc.order_tag, + }) + } +} + +impl fmt::Debug for Resp +where + M: Message, + ::Response: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "Resp[{:?}", self.msg)?; + match &self.stream { + AttachedStream::None => write!(f, "]"), + AttachedStream::Fixed(b) => write!(f, "; stream=buf:{}]", b.len()), + AttachedStream::Stream(_) => write!(f, "; stream]"), + } + } +} + +// ---- + +pub(crate) enum AttachedStream { + None, + Fixed(Bytes), + Stream(ByteStream), +} + +impl AttachedStream { + pub fn into_stream(self) -> Option { + match self { + AttachedStream::None => None, + AttachedStream::Fixed(b) => Some(Box::pin(futures::stream::once(async move { Ok(b) }))), + AttachedStream::Stream(s) => Some(s), + } + } +} + +// ---- ---- + +/// Encoding for requests into a ByteStream: +/// - priority: u8 +/// - path length: u8 +/// - path: [u8; path length] +/// - telemetry id length: u8 +/// - telemetry id: [u8; telemetry id length] +/// - msg len: u32 +/// - msg [u8; ..] +/// - the attached stream as the rest of the encoded stream +pub(crate) struct ReqEnc { + pub(crate) prio: RequestPriority, + pub(crate) path: Bytes, + pub(crate) telemetry_id: Bytes, + pub(crate) msg: Bytes, + pub(crate) stream: Option, + pub(crate) order_tag: Option, +} + +impl ReqEnc { + pub(crate) fn encode(self) -> (ByteStream, Option) { + let mut buf = BytesMut::with_capacity( + self.path.len() + self.telemetry_id.len() + self.msg.len() + 16, + ); + + buf.put_u8(self.prio); + + buf.put_u8(self.path.len() as u8); + buf.put(self.path); + + buf.put_u8(self.telemetry_id.len() as u8); + buf.put(&self.telemetry_id[..]); + + buf.put_u32(self.msg.len() as u32); + + let header = buf.freeze(); + + let res_stream: ByteStream = if let Some(stream) = self.stream { + Box::pin(futures::stream::iter([Ok(header), Ok(self.msg)]).chain(stream)) + } else { + Box::pin(futures::stream::iter([Ok(header), Ok(self.msg)])) + }; + (res_stream, self.order_tag) + } + + pub(crate) async fn decode(stream: ByteStream) -> Result { + Self::decode_aux(stream) + .await + .map_err(read_exact_error_to_error) + } + + async fn decode_aux(stream: ByteStream) -> Result { + let mut reader = ByteStreamReader::new(stream); + + let prio = reader.read_u8().await?; + + let path_len = reader.read_u8().await?; + let path = reader.read_exact(path_len as usize).await?; + + let telemetry_id_len = reader.read_u8().await?; + let telemetry_id = reader.read_exact(telemetry_id_len as usize).await?; + + let msg_len = reader.read_u32().await?; + let msg = reader.read_exact(msg_len as usize).await?; + + Ok(Self { + prio, + path, + telemetry_id, + msg, + stream: Some(reader.into_stream()), + order_tag: None, + }) + } +} + +/// Encoding for responses into a ByteStream: +/// IF SUCCESS: +/// - 0: u8 +/// - msg len: u32 +/// - msg [u8; ..] +/// - the attached stream as the rest of the encoded stream +/// IF ERROR: +/// - message length + 1: u8 +/// - error code: u8 +/// - message: [u8; message_length] +pub(crate) struct RespEnc { + msg: Bytes, + stream: Option, + order_tag: Option, +} + +impl RespEnc { + pub(crate) fn encode(resp: Result) -> (ByteStream, Option) { + match resp { + Ok(Self { + msg, + stream, + order_tag, + }) => { + let mut buf = BytesMut::with_capacity(4); + buf.put_u32(msg.len() as u32); + let header = buf.freeze(); + + let res_stream: ByteStream = if let Some(stream) = stream { + Box::pin(futures::stream::iter([Ok(header), Ok(msg)]).chain(stream)) + } else { + Box::pin(futures::stream::iter([Ok(header), Ok(msg)])) + }; + (res_stream, order_tag) + } + Err(err) => { + let err = std::io::Error::new( + std::io::ErrorKind::Other, + format!("netapp error: {}", err), + ); + ( + Box::pin(futures::stream::once(async move { Err(err) })), + None, + ) + } + } + } + + pub(crate) async fn decode(stream: ByteStream) -> Result { + Self::decode_aux(stream) + .await + .map_err(read_exact_error_to_error) + } + + async fn decode_aux(stream: ByteStream) -> Result { + let mut reader = ByteStreamReader::new(stream); + + let msg_len = reader.read_u32().await?; + let msg = reader.read_exact(msg_len as usize).await?; + + // Check whether the response stream still has data or not. + // If no more data is coming, this will defuse the request canceller. + // If we didn't do this, and the client doesn't try to read from the stream, + // the request canceller doesn't know that we read everything and + // sends a cancellation message to the server (which they don't care about). + reader.fill_buffer().await; + + Ok(Self { + msg, + stream: Some(reader.into_stream()), + order_tag: None, + }) + } +} + +fn read_exact_error_to_error(e: ReadExactError) -> Error { + match e { + ReadExactError::Stream(err) => Error::Remote(err.kind(), err.to_string()), + ReadExactError::UnexpectedEos => Error::Framing, + } +} diff --git a/src/net/netapp.rs b/src/net/netapp.rs new file mode 100644 index 00000000..36c6fc88 --- /dev/null +++ b/src/net/netapp.rs @@ -0,0 +1,470 @@ +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, RwLock}; + +use log::{debug, error, info, trace, warn}; + +use arc_swap::ArcSwapOption; + +use serde::{Deserialize, Serialize}; +use sodiumoxide::crypto::auth; +use sodiumoxide::crypto::sign::ed25519; + +use futures::stream::futures_unordered::FuturesUnordered; +use futures::stream::StreamExt; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::select; +use tokio::sync::{mpsc, watch}; + +use crate::client::*; +use crate::endpoint::*; +use crate::error::*; +use crate::message::*; +use crate::server::*; + +/// A node's identifier, which is also its public cryptographic key +pub type NodeID = sodiumoxide::crypto::sign::ed25519::PublicKey; +/// A node's secret key +pub type NodeKey = sodiumoxide::crypto::sign::ed25519::SecretKey; +/// A network key +pub type NetworkKey = sodiumoxide::crypto::auth::Key; + +/// Tag which is exchanged between client and server upon connection establishment +/// to check that they are running compatible versions of Netapp, +/// composed of 8 bytes for Netapp version and 8 bytes for client version +pub(crate) type VersionTag = [u8; 16]; + +/// Value of garage_net version used in the version tag +/// We are no longer using prefix `netapp` as garage_net is forked from the netapp crate. +/// Since Garage v1.0, we have replaced the prefix by `grgnet` (shorthand for garage_net). +pub(crate) const NETAPP_VERSION_TAG: u64 = 0x6772676e65740010; // grgnet 0x0010 (1.0) + +/// HelloMessage is sent by the client on a Netapp connection to indicate +/// that they are also a server and ready to receive incoming connections +/// at the specified address and port. If the client doesn't know their +/// public address, they don't need to specify it and we look at the +/// remote address of the socket is used instead. +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct HelloMessage { + pub server_addr: Option, + pub server_port: u16, +} + +impl Message for HelloMessage { + type Response = (); +} + +type OnConnectHandler = Box; +type OnDisconnectHandler = Box; + +/// NetApp is the main class that handles incoming and outgoing connections. +/// +/// NetApp can be used in a stand-alone fashion or together with a peering strategy. +/// If using it alone, you will want to set `on_connect` and `on_disconnect` events +/// in order to manage information about the current peer list. +pub struct NetApp { + bind_outgoing_to: Option, + listen_params: ArcSwapOption, + + /// Version tag, 8 bytes for netapp version, 8 bytes for app version + pub version_tag: VersionTag, + /// Network secret key + pub netid: auth::Key, + /// Our peer ID + pub id: NodeID, + /// Private key associated with our peer ID + pub privkey: ed25519::SecretKey, + + pub(crate) server_conns: RwLock>>, + pub(crate) client_conns: RwLock>>, + + pub(crate) endpoints: RwLock>, + hello_endpoint: ArcSwapOption>, + + on_connected_handler: ArcSwapOption, + on_disconnected_handler: ArcSwapOption, +} + +struct ListenParams { + listen_addr: SocketAddr, + public_addr: Option, +} + +impl NetApp { + /// Creates a new instance of NetApp, which can serve either as a full p2p node, + /// or just as a passive client. To upgrade to a full p2p node, spawn a listener + /// using `.listen()` + /// + /// Our Peer ID is the public key associated to the secret key given here. + pub fn new( + app_version_tag: u64, + netid: auth::Key, + privkey: ed25519::SecretKey, + bind_outgoing_to: Option, + ) -> Arc { + let mut version_tag = [0u8; 16]; + version_tag[0..8].copy_from_slice(&u64::to_be_bytes(NETAPP_VERSION_TAG)[..]); + version_tag[8..16].copy_from_slice(&u64::to_be_bytes(app_version_tag)[..]); + + let id = privkey.public_key(); + let netapp = Arc::new(Self { + bind_outgoing_to, + listen_params: ArcSwapOption::new(None), + version_tag, + netid, + id, + privkey, + server_conns: RwLock::new(HashMap::new()), + client_conns: RwLock::new(HashMap::new()), + endpoints: RwLock::new(HashMap::new()), + hello_endpoint: ArcSwapOption::new(None), + on_connected_handler: ArcSwapOption::new(None), + on_disconnected_handler: ArcSwapOption::new(None), + }); + + netapp + .hello_endpoint + .swap(Some(netapp.endpoint("garage_net/netapp.rs/Hello".into()))); + netapp + .hello_endpoint + .load_full() + .unwrap() + .set_handler(netapp.clone()); + + netapp + } + + /// Set the handler to be called when a new connection (incoming or outgoing) has + /// been successfully established. Do not set this if using a peering strategy, + /// as the peering strategy will need to set this itself. + pub fn on_connected(&self, handler: F) + where + F: Fn(NodeID, SocketAddr, bool) + Sized + Send + Sync + 'static, + { + self.on_connected_handler + .store(Some(Arc::new(Box::new(handler)))); + } + + /// Set the handler to be called when an existing connection (incoming or outgoing) has + /// been closed by either party. Do not set this if using a peering strategy, + /// as the peering strategy will need to set this itself. + pub fn on_disconnected(&self, handler: F) + where + F: Fn(NodeID, bool) + Sized + Send + Sync + 'static, + { + self.on_disconnected_handler + .store(Some(Arc::new(Box::new(handler)))); + } + + /// Create a new endpoint with path `path`, + /// that handles messages of type `M`. + /// `H` is the type of the object that should handle requests + /// to this endpoint on the local node. If you don't want + /// to handle request on the local node (e.g. if this node + /// is only a client in the network), define the type `H` + /// to be `()`. + /// This function will panic if the endpoint has already been + /// created. + pub fn endpoint(self: &Arc, path: String) -> Arc> + where + M: Message + 'static, + H: StreamingEndpointHandler + 'static, + { + let endpoint = Arc::new(Endpoint::::new(self.clone(), path.clone())); + let endpoint_arc = EndpointArc(endpoint.clone()); + if self + .endpoints + .write() + .unwrap() + .insert(path.clone(), Box::new(endpoint_arc)) + .is_some() + { + panic!("Redefining endpoint: {}", path); + }; + endpoint + } + + /// Main listening process for our app. This future runs during the whole + /// run time of our application. + /// If this is not called, the NetApp instance remains a passive client. + pub async fn listen( + self: Arc, + listen_addr: SocketAddr, + public_addr: Option, + mut must_exit: watch::Receiver, + ) { + let listen_params = ListenParams { + listen_addr, + public_addr, + }; + if self + .listen_params + .swap(Some(Arc::new(listen_params))) + .is_some() + { + error!("Trying to listen on NetApp but we're already listening!"); + } + + let listener = TcpListener::bind(listen_addr).await.unwrap(); + info!("Listening on {}", listen_addr); + + let (conn_in, mut conn_out) = mpsc::unbounded_channel(); + let connection_collector = tokio::spawn(async move { + let mut collection = FuturesUnordered::new(); + loop { + if collection.is_empty() { + match conn_out.recv().await { + Some(f) => collection.push(f), + None => break, + } + } else { + select! { + new_fut = conn_out.recv() => { + match new_fut { + Some(f) => collection.push(f), + None => break, + } + } + result = collection.next() => { + trace!("Collected connection: {:?}", result); + } + } + } + } + debug!("Collecting last open server connections."); + while let Some(conn_res) = collection.next().await { + trace!("Collected connection: {:?}", conn_res); + } + debug!("No more server connections to collect"); + }); + + while !*must_exit.borrow_and_update() { + let (socket, peer_addr) = select! { + sockres = listener.accept() => { + match sockres { + Ok(x) => x, + Err(e) => { + warn!("Error in listener.accept: {}", e); + continue; + } + } + }, + _ = must_exit.changed() => continue, + }; + + info!( + "Incoming connection from {}, negotiating handshake...", + peer_addr + ); + let self2 = self.clone(); + let must_exit2 = must_exit.clone(); + conn_in + .send(tokio::spawn(async move { + ServerConn::run(self2, socket, must_exit2) + .await + .log_err("ServerConn::run"); + })) + .log_err("Failed to send connection to connection collector"); + } + + drop(conn_in); + + connection_collector + .await + .log_err("Failed to await for connection collector"); + } + + /// Drop all endpoint handlers, as well as handlers for connection/disconnection + /// events. (This disables the peering strategy) + /// + /// Use this when terminating to break reference cycles + pub fn drop_all_handlers(&self) { + for (_, endpoint) in self.endpoints.read().unwrap().iter() { + endpoint.drop_handler(); + } + self.on_connected_handler.store(None); + self.on_disconnected_handler.store(None); + } + + /// Attempt to connect to a peer, given by its ip:port and its public key. + /// The public key will be checked during the secret handshake process. + /// This function returns once the connection has been established and a + /// successful handshake was made. At this point we can send messages to + /// the other node with `Netapp::request` + pub async fn try_connect(self: Arc, ip: SocketAddr, id: NodeID) -> Result<(), Error> { + // Don't connect to ourself, we don't care + if id == self.id { + return Ok(()); + } + + // Don't connect if already connected + if self.client_conns.read().unwrap().contains_key(&id) { + return Ok(()); + } + + let stream = match self.bind_outgoing_to { + Some(addr) => { + let socket = if addr.is_ipv4() { + TcpSocket::new_v4()? + } else { + TcpSocket::new_v6()? + }; + socket.bind(SocketAddr::new(addr, 0))?; + socket.connect(ip).await? + } + None => TcpStream::connect(ip).await?, + }; + info!("Connected to {}, negotiating handshake...", ip); + ClientConn::init(self, stream, id).await?; + Ok(()) + } + + /// Close the outgoing connection we have to a node specified by its public key, + /// if such a connection is currently open. + pub fn disconnect(self: &Arc, id: &NodeID) { + let conn = self.client_conns.write().unwrap().remove(id); + + // If id is ourself, we're not supposed to have a connection open + if *id == self.id { + // sanity check + assert!(conn.is_none(), "had a connection to local node"); + return; + } + + if let Some(c) = conn { + debug!( + "Closing connection to {} ({})", + hex::encode(&c.peer_id[..8]), + c.remote_addr + ); + c.close(); + + // call on_disconnected_handler immediately, since the connection was removed + let id = *id; + let self2 = self.clone(); + tokio::spawn(async move { + if let Some(h) = self2.on_disconnected_handler.load().as_ref() { + h(id, false); + } + }); + } + } + + // Called from conn.rs when an incoming connection is successfully established + // Registers the connection in our list of connections + // Do not yet call the on_connected handler, because we don't know if the remote + // has an actual IP address and port we can call them back on. + // We will know this when they send a Hello message, which is handled below. + pub(crate) fn connected_as_server(&self, id: NodeID, conn: Arc) { + info!( + "Accepted connection from {} at {}", + hex::encode(&id[..8]), + conn.remote_addr + ); + + self.server_conns.write().unwrap().insert(id, conn); + } + + // Handle hello message from a client. This message is used for them to tell us + // that they are listening on a certain port number on which we can call them back. + // At this point we know they are a full network member, and not just a client, + // and we call the on_connected handler so that the peering strategy knows + // we have a new potential peer + + // Called from conn.rs when an incoming connection is closed. + // We deregister the connection from server_conns and call the + // handler registered by on_disconnected + pub(crate) fn disconnected_as_server(&self, id: &NodeID, conn: Arc) { + info!("Connection from {} closed", hex::encode(&id[..8])); + + let mut conn_list = self.server_conns.write().unwrap(); + if let Some(c) = conn_list.get(id) { + if Arc::ptr_eq(c, &conn) { + conn_list.remove(id); + drop(conn_list); + + if let Some(h) = self.on_disconnected_handler.load().as_ref() { + h(conn.peer_id, true); + } + } + } + } + + // Called from conn.rs when an outgoinc connection is successfully established. + // The connection is registered in self.client_conns, and the + // on_connected handler is called. + // + // Since we are ourself listening, we send them a Hello message so that + // they know on which port to call us back. (TODO: don't do this if we are + // just a simple client and not a full p2p node) + pub(crate) fn connected_as_client(&self, id: NodeID, conn: Arc) { + info!("Connection established to {}", hex::encode(&id[..8])); + + { + let old_c_opt = self.client_conns.write().unwrap().insert(id, conn.clone()); + if let Some(old_c) = old_c_opt { + tokio::spawn(async move { old_c.close() }); + } + } + + if let Some(h) = self.on_connected_handler.load().as_ref() { + h(conn.peer_id, conn.remote_addr, false); + } + + if let Some(lp) = self.listen_params.load_full() { + let server_addr = lp.public_addr.map(|x| x.ip()); + let server_port = lp + .public_addr + .map(|x| x.port()) + .unwrap_or(lp.listen_addr.port()); + let hello_endpoint = self.hello_endpoint.load_full().unwrap(); + tokio::spawn(async move { + hello_endpoint + .call( + &conn.peer_id, + HelloMessage { + server_addr, + server_port, + }, + PRIO_NORMAL, + ) + .await + .map(|_| ()) + .log_err("Sending hello message"); + }); + } + } + + // Called from conn.rs when an outgoinc connection is closed. + // The connection is removed from conn_list, and the on_disconnected handler + // is called. + pub(crate) fn disconnected_as_client(&self, id: &NodeID, conn: Arc) { + info!("Connection to {} closed", hex::encode(&id[..8])); + let mut conn_list = self.client_conns.write().unwrap(); + if let Some(c) = conn_list.get(id) { + if Arc::ptr_eq(c, &conn) { + conn_list.remove(id); + drop(conn_list); + + if let Some(h) = self.on_disconnected_handler.load().as_ref() { + h(conn.peer_id, false); + } + } + } + // else case: happens if connection was removed in .disconnect() + // in which case on_disconnected_handler was already called + } +} + +impl EndpointHandler for NetApp { + async fn handle(self: &Arc, msg: &HelloMessage, from: NodeID) { + debug!("Hello from {:?}: {:?}", hex::encode(&from[..8]), msg); + if let Some(h) = self.on_connected_handler.load().as_ref() { + if let Some(c) = self.server_conns.read().unwrap().get(&from) { + let remote_ip = msg.server_addr.unwrap_or_else(|| c.remote_addr.ip()); + let remote_addr = SocketAddr::new(remote_ip, msg.server_port); + h(from, remote_addr, true); + } + } + } +} diff --git a/src/net/peering.rs b/src/net/peering.rs new file mode 100644 index 00000000..08378a08 --- /dev/null +++ b/src/net/peering.rs @@ -0,0 +1,615 @@ +use std::collections::{HashMap, VecDeque}; +use std::net::SocketAddr; +use std::sync::atomic::{self, AtomicU64}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwap; +use log::{debug, info, trace, warn}; +use serde::{Deserialize, Serialize}; + +use tokio::select; +use tokio::sync::watch; + +use sodiumoxide::crypto::hash; + +use crate::endpoint::*; +use crate::error::*; +use crate::netapp::*; + +use crate::message::*; +use crate::NodeID; + +const CONN_RETRY_INTERVAL: Duration = Duration::from_secs(30); +const CONN_MAX_RETRIES: usize = 10; +const PING_INTERVAL: Duration = Duration::from_secs(15); +const LOOP_DELAY: Duration = Duration::from_secs(1); +const FAILED_PING_THRESHOLD: usize = 4; + +const DEFAULT_PING_TIMEOUT_MILLIS: u64 = 10_000; + +// -- Protocol messages -- + +#[derive(Serialize, Deserialize)] +struct PingMessage { + pub id: u64, + pub peer_list_hash: hash::Digest, +} + +impl Message for PingMessage { + type Response = PingMessage; +} + +#[derive(Serialize, Deserialize)] +struct PeerListMessage { + pub list: Vec<(NodeID, SocketAddr)>, +} + +impl Message for PeerListMessage { + type Response = PeerListMessage; +} + +// -- Algorithm data structures -- + +#[derive(Debug)] +struct PeerInfoInternal { + // known_addrs contains all of the addresses everyone gave us + known_addrs: Vec, + + state: PeerConnState, + last_send_ping: Option, + last_seen: Option, + ping: VecDeque, + failed_pings: usize, +} + +impl PeerInfoInternal { + fn new(state: PeerConnState, known_addr: Option) -> Self { + Self { + known_addrs: known_addr.map(|x| vec![x]).unwrap_or_default(), + state, + last_send_ping: None, + last_seen: None, + ping: VecDeque::new(), + failed_pings: 0, + } + } + fn add_addr(&mut self, addr: SocketAddr) -> bool { + if !self.known_addrs.contains(&addr) { + self.known_addrs.push(addr); + // If we are learning a new address for this node, + // we want to retry connecting + self.state = match self.state { + PeerConnState::Trying(_) => PeerConnState::Trying(0), + PeerConnState::Waiting(_, _) | PeerConnState::Abandonned => { + PeerConnState::Waiting(0, Instant::now()) + } + x @ (PeerConnState::Ourself | PeerConnState::Connected { .. }) => x, + }; + true + } else { + false + } + } +} + +/// Information that the full mesh peering strategy can return about the peers it knows of +#[derive(Copy, Clone, Debug)] +pub struct PeerInfo { + /// The node's identifier (its public key) + pub id: NodeID, + /// The current status of our connection to this node + pub state: PeerConnState, + /// The last time at which the node was seen + pub last_seen: Option, + /// The average ping to this node on recent observations (if at least one ping value is known) + pub avg_ping: Option, + /// The maximum observed ping to this node on recent observations (if at least one + /// ping value is known) + pub max_ping: Option, + /// The median ping to this node on recent observations (if at least one ping value + /// is known) + pub med_ping: Option, +} + +impl PeerInfo { + /// Returns true if we can currently send requests to this peer + pub fn is_up(&self) -> bool { + self.state.is_up() + } +} + +/// PeerConnState: possible states for our tentative connections to given peer +/// This structure is only interested in recording connection info for outgoing +/// TCP connections +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum PeerConnState { + /// This entry represents ourself (the local node) + Ourself, + + /// We currently have a connection to this peer + Connected { addr: SocketAddr }, + + /// Our next connection tentative (the nth, where n is the first value of the tuple) + /// will be at given Instant + Waiting(usize, Instant), + + /// A connection tentative is in progress (the nth, where n is the value stored) + Trying(usize), + + /// We abandoned trying to connect to this peer (too many failed attempts) + Abandonned, +} + +impl PeerConnState { + /// Returns true if we can currently send requests to this peer + pub fn is_up(&self) -> bool { + matches!(self, Self::Ourself | Self::Connected { .. }) + } +} + +struct KnownHosts { + list: HashMap, + hash: hash::Digest, +} + +impl KnownHosts { + fn new() -> Self { + let list = HashMap::new(); + let mut ret = Self { + list, + hash: hash::Digest::from_slice(&[0u8; 64][..]).unwrap(), + }; + ret.update_hash(); + ret + } + fn update_hash(&mut self) { + // The hash is a value that is exchanged between nodes when they ping one + // another. Nodes compare their known hosts hash to know if they are connected + // to the same set of nodes. If the hashes differ, they are connected to + // different nodes and they trigger an exchange of the full list of active + // connections. The hash value only represents the set of node IDs and not + // their actual socket addresses, because nodes can be connected via different + // addresses and that shouldn't necessarily trigger a full peer exchange. + let mut list = self + .list + .iter() + .filter(|(_, peer)| peer.state.is_up()) + .map(|(id, _)| *id) + .collect::>(); + list.sort(); + let mut hash_state = hash::State::new(); + for id in list { + hash_state.update(&id[..]); + } + self.hash = hash_state.finalize(); + } + fn connected_peers_vec(&self) -> Vec<(NodeID, SocketAddr)> { + self.list + .iter() + .filter_map(|(id, peer)| match peer.state { + PeerConnState::Connected { addr } => Some((*id, addr)), + _ => None, + }) + .collect::>() + } +} + +/// A "Full Mesh" peering strategy is a peering strategy that tries +/// to establish and maintain a direct connection with all of the +/// known nodes in the network. +pub struct PeeringManager { + netapp: Arc, + known_hosts: RwLock, + public_peer_list: ArcSwap>, + + next_ping_id: AtomicU64, + ping_endpoint: Arc>, + peer_list_endpoint: Arc>, + + ping_timeout_millis: AtomicU64, +} + +impl PeeringManager { + /// Create a new Full Mesh peering strategy. + /// The strategy will not be run until `.run()` is called and awaited. + /// Once that happens, the peering strategy will try to connect + /// to all of the nodes specified in the bootstrap list. + pub fn new( + netapp: Arc, + bootstrap_list: Vec<(NodeID, SocketAddr)>, + our_addr: Option, + ) -> Arc { + let mut known_hosts = KnownHosts::new(); + for (id, addr) in bootstrap_list { + if id != netapp.id { + known_hosts.list.insert( + id, + PeerInfoInternal::new(PeerConnState::Waiting(0, Instant::now()), Some(addr)), + ); + } + } + + known_hosts.list.insert( + netapp.id, + PeerInfoInternal::new(PeerConnState::Ourself, our_addr), + ); + known_hosts.update_hash(); + + let strat = Arc::new(Self { + netapp: netapp.clone(), + known_hosts: RwLock::new(known_hosts), + public_peer_list: ArcSwap::new(Arc::new(Vec::new())), + next_ping_id: AtomicU64::new(42), + ping_endpoint: netapp.endpoint("garage_net/peering.rs/Ping".into()), + peer_list_endpoint: netapp.endpoint("garage_net/peering.rs/PeerList".into()), + ping_timeout_millis: DEFAULT_PING_TIMEOUT_MILLIS.into(), + }); + + strat.update_public_peer_list(&strat.known_hosts.read().unwrap()); + + strat.ping_endpoint.set_handler(strat.clone()); + strat.peer_list_endpoint.set_handler(strat.clone()); + + let strat2 = strat.clone(); + netapp.on_connected(move |id: NodeID, addr: SocketAddr, is_incoming: bool| { + strat2.on_connected(id, addr, is_incoming); + }); + + let strat2 = strat.clone(); + netapp.on_disconnected(move |id: NodeID, is_incoming: bool| { + strat2.on_disconnected(id, is_incoming); + }); + + strat + } + + /// Run the full mesh peering strategy. + /// This future exits when the `must_exit` watch becomes true. + pub async fn run(self: Arc, must_exit: watch::Receiver) { + while !*must_exit.borrow() { + // 1. Read current state: get list of connected peers (ping them) + let (to_ping, to_retry) = { + let known_hosts = self.known_hosts.read().unwrap(); + trace!("known_hosts: {} peers", known_hosts.list.len()); + + let mut to_ping = vec![]; + let mut to_retry = vec![]; + for (id, info) in known_hosts.list.iter() { + trace!("{}, {:?}", hex::encode(&id[..8]), info); + match info.state { + PeerConnState::Connected { .. } => { + let must_ping = match info.last_send_ping { + None => true, + Some(t) => Instant::now() - t > PING_INTERVAL, + }; + if must_ping { + to_ping.push(*id); + } + } + PeerConnState::Waiting(_, t) => { + if Instant::now() >= t { + to_retry.push(*id); + } + } + _ => (), + } + } + (to_ping, to_retry) + }; + + // 2. Dispatch ping to hosts + trace!("to_ping: {} peers", to_ping.len()); + if !to_ping.is_empty() { + let mut known_hosts = self.known_hosts.write().unwrap(); + for id in to_ping.iter() { + known_hosts.list.get_mut(id).unwrap().last_send_ping = Some(Instant::now()); + } + drop(known_hosts); + for id in to_ping { + tokio::spawn(self.clone().ping(id)); + } + } + + // 3. Try reconnects + trace!("to_retry: {} peers", to_retry.len()); + if !to_retry.is_empty() { + let mut known_hosts = self.known_hosts.write().unwrap(); + for id in to_retry { + if let Some(h) = known_hosts.list.get_mut(&id) { + if let PeerConnState::Waiting(i, _) = h.state { + info!( + "Retrying connection to {} at {} ({})", + hex::encode(&id[..8]), + h.known_addrs + .iter() + .map(|x| format!("{}", x)) + .collect::>() + .join(", "), + i + 1 + ); + h.state = PeerConnState::Trying(i); + + let addresses = h.known_addrs.clone(); + tokio::spawn(self.clone().try_connect(id, addresses)); + } + } + } + self.update_public_peer_list(&known_hosts); + } + + // 4. Sleep before next loop iteration + tokio::time::sleep(LOOP_DELAY).await; + } + } + + /// Returns a list of currently known peers in the network. + pub fn get_peer_list(&self) -> Arc> { + self.public_peer_list.load_full() + } + + /// Set the timeout for ping messages, in milliseconds + pub fn set_ping_timeout_millis(&self, timeout: u64) { + self.ping_timeout_millis + .store(timeout, atomic::Ordering::Relaxed); + } + + // -- internal stuff -- + + fn update_public_peer_list(&self, known_hosts: &KnownHosts) { + let mut pub_peer_list = Vec::with_capacity(known_hosts.list.len()); + for (id, info) in known_hosts.list.iter() { + if *id == self.netapp.id { + // sanity check + assert!(matches!(info.state, PeerConnState::Ourself)); + } + let mut pings = info.ping.iter().cloned().collect::>(); + pings.sort(); + if !pings.is_empty() { + pub_peer_list.push(PeerInfo { + id: *id, + state: info.state, + last_seen: info.last_seen, + avg_ping: Some(pings.iter().sum::().div_f64(pings.len() as f64)), + max_ping: pings.last().cloned(), + med_ping: Some(pings[pings.len() / 2]), + }); + } else { + pub_peer_list.push(PeerInfo { + id: *id, + state: info.state, + last_seen: info.last_seen, + avg_ping: None, + max_ping: None, + med_ping: None, + }); + } + } + self.public_peer_list.store(Arc::new(pub_peer_list)); + } + + async fn ping(self: Arc, id: NodeID) { + let peer_list_hash = self.known_hosts.read().unwrap().hash; + let ping_id = self.next_ping_id.fetch_add(1u64, atomic::Ordering::Relaxed); + let ping_time = Instant::now(); + let ping_timeout = + Duration::from_millis(self.ping_timeout_millis.load(atomic::Ordering::Relaxed)); + let ping_msg = PingMessage { + id: ping_id, + peer_list_hash, + }; + + debug!( + "Sending ping {} to {} at {:?}", + ping_id, + hex::encode(&id[..8]), + ping_time + ); + let ping_response = select! { + r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH) => r, + _ = tokio::time::sleep(ping_timeout) => Err(Error::Message("Ping timeout".into())), + }; + + match ping_response { + Err(e) => { + warn!("Error pinging {}: {}", hex::encode(&id[..8]), e); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.failed_pings += 1; + if host.failed_pings > FAILED_PING_THRESHOLD { + warn!( + "Too many failed pings from {}, closing connection.", + hex::encode(&id[..8]) + ); + // this will later update info in known_hosts + // through the disconnection handler + self.netapp.disconnect(&id); + } + } + } + Ok(ping_resp) => { + let resp_time = Instant::now(); + debug!( + "Got ping response from {} at {:?}", + hex::encode(&id[..8]), + resp_time + ); + { + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.failed_pings = 0; + host.last_seen = Some(resp_time); + host.ping.push_back(resp_time - ping_time); + while host.ping.len() > 10 { + host.ping.pop_front(); + } + self.update_public_peer_list(&known_hosts); + } + } + if ping_resp.peer_list_hash != peer_list_hash { + self.exchange_peers(&id).await; + } + } + } + } + + async fn exchange_peers(self: Arc, id: &NodeID) { + let peer_list = self.known_hosts.read().unwrap().connected_peers_vec(); + let pex_message = PeerListMessage { list: peer_list }; + match self + .peer_list_endpoint + .call(id, pex_message, PRIO_BACKGROUND) + .await + { + Err(e) => warn!("Error doing peer exchange: {}", e), + Ok(resp) => { + self.handle_peer_list(&resp.list[..]); + } + } + } + + fn handle_peer_list(&self, list: &[(NodeID, SocketAddr)]) { + let mut known_hosts = self.known_hosts.write().unwrap(); + + let mut changed = false; + for (id, addr) in list.iter() { + if let Some(kh) = known_hosts.list.get_mut(id) { + if kh.add_addr(*addr) { + changed = true; + } + } else { + known_hosts.list.insert(*id, self.new_peer(id, *addr)); + changed = true; + } + } + + if changed { + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + } + + async fn try_connect(self: Arc, id: NodeID, addresses: Vec) { + let conn_addr = { + let mut ret = None; + for addr in addresses.iter() { + debug!("Trying address {} for peer {}", addr, hex::encode(&id[..8])); + match self.netapp.clone().try_connect(*addr, id).await { + Ok(()) => { + ret = Some(*addr); + break; + } + Err(e) => { + debug!( + "Error connecting to {} at {}: {}", + hex::encode(&id[..8]), + addr, + e + ); + } + } + } + ret + }; + + if let Some(ok_addr) = conn_addr { + self.on_connected(id, ok_addr, false); + } else { + warn!( + "Could not connect to peer {} ({} addresses tried)", + hex::encode(&id[..8]), + addresses.len() + ); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = match host.state { + PeerConnState::Trying(i) => { + if i >= CONN_MAX_RETRIES { + PeerConnState::Abandonned + } else { + PeerConnState::Waiting(i + 1, Instant::now() + CONN_RETRY_INTERVAL) + } + } + _ => PeerConnState::Waiting(0, Instant::now() + CONN_RETRY_INTERVAL), + }; + self.update_public_peer_list(&known_hosts); + } + } + } + + fn on_connected(self: &Arc, id: NodeID, addr: SocketAddr, is_incoming: bool) { + if id == self.netapp.id { + // sanity check + panic!( + "on_connected from local node, id={:?}, addr={}, incoming={}", + id, addr, is_incoming + ); + } + + let mut known_hosts = self.known_hosts.write().unwrap(); + if is_incoming { + if let Some(host) = known_hosts.list.get_mut(&id) { + host.add_addr(addr); + } else { + known_hosts.list.insert(id, self.new_peer(&id, addr)); + } + } else { + info!( + "Successfully connected to {} at {}", + hex::encode(&id[..8]), + addr + ); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = PeerConnState::Connected { addr }; + host.add_addr(addr); + } else { + known_hosts.list.insert( + id, + PeerInfoInternal::new(PeerConnState::Connected { addr }, Some(addr)), + ); + } + } + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + + fn on_disconnected(self: &Arc, id: NodeID, is_incoming: bool) { + if !is_incoming { + info!("Connection to {} was closed", hex::encode(&id[..8])); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = PeerConnState::Waiting(0, Instant::now()); + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + } + } + + fn new_peer(&self, id: &NodeID, addr: SocketAddr) -> PeerInfoInternal { + assert!(*id != self.netapp.id); + PeerInfoInternal::new(PeerConnState::Waiting(0, Instant::now()), Some(addr)) + } +} + +impl EndpointHandler for PeeringManager { + async fn handle(self: &Arc, ping: &PingMessage, from: NodeID) -> PingMessage { + let ping_resp = PingMessage { + id: ping.id, + peer_list_hash: self.known_hosts.read().unwrap().hash, + }; + debug!("Ping from {}", hex::encode(&from[..8])); + ping_resp + } +} + +impl EndpointHandler for PeeringManager { + async fn handle( + self: &Arc, + peer_list: &PeerListMessage, + _from: NodeID, + ) -> PeerListMessage { + self.handle_peer_list(&peer_list.list[..]); + let peer_list = self.known_hosts.read().unwrap().connected_peers_vec(); + PeerListMessage { list: peer_list } + } +} diff --git a/src/net/recv.rs b/src/net/recv.rs new file mode 100644 index 00000000..35a6d71a --- /dev/null +++ b/src/net/recv.rs @@ -0,0 +1,151 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use bytes::Bytes; +use log::*; + +use futures::AsyncReadExt; +use tokio::sync::mpsc; + +use crate::error::*; +use crate::send::*; +use crate::stream::*; + +/// Structure to warn when the sender is dropped before end of stream was reached, like when +/// connection to some remote drops while transmitting data +struct Sender { + inner: Option>, +} + +impl Sender { + fn new(inner: mpsc::UnboundedSender) -> Self { + Sender { inner: Some(inner) } + } + + fn send(&self, packet: Packet) { + let _ = self.inner.as_ref().unwrap().send(packet); + } + + fn end(&mut self) { + self.inner = None; + } +} + +impl Drop for Sender { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + let _ = inner.send(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "Netapp connection dropped before end of stream", + ))); + } + } +} + +/// The RecvLoop trait, which is implemented both by the client and the server +/// connection objects (ServerConn and ClientConn) adds a method `.recv_loop()` +/// and a prototype of a handler for received messages `.recv_handler()` that +/// must be filled by implementors. `.recv_loop()` receives messages in a loop +/// according to the protocol defined above: chunks of message in progress of being +/// received are stored in a buffer, and when the last chunk of a message is received, +/// the full message is passed to the receive handler. +pub(crate) trait RecvLoop: Sync + 'static { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream); + fn cancel_handler(self: &Arc, _id: RequestID) {} + + async fn recv_loop(self: Arc, mut read: R, debug_name: String) -> Result<(), Error> + where + R: AsyncReadExt + Unpin + Send + Sync, + { + let mut streams: HashMap = HashMap::new(); + loop { + trace!( + "recv_loop({}): in_progress = {:?}", + debug_name, + streams.iter().map(|(id, _)| id).collect::>() + ); + + let mut header_id = [0u8; RequestID::BITS as usize / 8]; + match read.read_exact(&mut header_id[..]).await { + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + }; + let id = RequestID::from_be_bytes(header_id); + + let mut header_size = [0u8; ChunkLength::BITS as usize / 8]; + read.read_exact(&mut header_size[..]).await?; + let size = ChunkLength::from_be_bytes(header_size); + + if size == CANCEL_REQUEST { + if let Some(mut stream) = streams.remove(&id) { + let _ = stream.send(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "netapp: cancel requested", + ))); + stream.end(); + } + self.cancel_handler(id); + continue; + } + + let has_cont = (size & CHUNK_FLAG_HAS_CONTINUATION) != 0; + let is_error = (size & CHUNK_FLAG_ERROR) != 0; + let size = (size & CHUNK_LENGTH_MASK) as usize; + let mut next_slice = vec![0; size as usize]; + read.read_exact(&mut next_slice[..]).await?; + + let packet = if is_error { + let kind = u8_to_io_errorkind(next_slice[0]); + let msg = + std::str::from_utf8(&next_slice[1..]).unwrap_or(""); + debug!( + "recv_loop({}): got id {}, error {:?}: {}", + debug_name, id, kind, msg + ); + Some(Err(std::io::Error::new(kind, msg.to_string()))) + } else { + trace!( + "recv_loop({}): got id {}, size {}, has_cont {}", + debug_name, + id, + size, + has_cont + ); + if !next_slice.is_empty() { + Some(Ok(Bytes::from(next_slice))) + } else { + None + } + }; + + let mut sender = if let Some(send) = streams.remove(&(id)) { + send + } else { + let (send, recv) = mpsc::unbounded_channel(); + trace!("recv_loop({}): id {} is new channel", debug_name, id); + self.recv_handler( + id, + Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(recv)), + ); + Sender::new(send) + }; + + if let Some(packet) = packet { + // If we cannot put packet in channel, it means that the + // receiving end of the channel is disconnected. + // We still need to reach eos before dropping this sender + let _ = sender.send(packet); + } + + if has_cont { + assert!(!is_error); + streams.insert(id, sender); + } else { + trace!("recv_loop({}): close channel id {}", debug_name, id); + sender.end(); + } + } + Ok(()) + } +} diff --git a/src/net/send.rs b/src/net/send.rs new file mode 100644 index 00000000..6f1ac02c --- /dev/null +++ b/src/net/send.rs @@ -0,0 +1,359 @@ +use std::collections::{HashMap, VecDeque}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use bytes::{BufMut, Bytes, BytesMut}; +use log::*; + +use futures::{AsyncWriteExt, Future}; +use kuska_handshake::async_std::BoxStreamWrite; +use tokio::sync::mpsc; + +use crate::error::*; +use crate::message::*; +use crate::stream::*; + +// Messages are sent by chunks +// Chunk format: +// - u32 BE: request id (same for request and response) +// - u16 BE: chunk length + flags: +// CHUNK_FLAG_HAS_CONTINUATION when this is not the last chunk of the stream +// CHUNK_FLAG_ERROR if this chunk denotes an error +// (these two flags are exclusive, an error denotes the end of the stream) +// **special value** 0xFFFF indicates a CANCEL message +// - [u8; chunk_length], either +// - if not error: chunk data +// - if error: +// - u8: error kind, encoded using error::io_errorkind_to_u8 +// - rest: error message +// - absent for cancel message + +pub(crate) type RequestID = u32; +pub(crate) type ChunkLength = u16; + +pub(crate) const MAX_CHUNK_LENGTH: ChunkLength = 0x3FF0; +pub(crate) const CHUNK_FLAG_ERROR: ChunkLength = 0x4000; +pub(crate) const CHUNK_FLAG_HAS_CONTINUATION: ChunkLength = 0x8000; +pub(crate) const CHUNK_LENGTH_MASK: ChunkLength = 0x3FFF; +pub(crate) const CANCEL_REQUEST: ChunkLength = 0xFFFF; + +pub(crate) enum SendItem { + Stream(RequestID, RequestPriority, Option, ByteStream), + Cancel(RequestID), +} + +// ---- + +struct SendQueue { + items: Vec<(u8, SendQueuePriority)>, +} + +struct SendQueuePriority { + items: VecDeque, + order: HashMap>, +} + +struct SendQueueItem { + id: RequestID, + prio: RequestPriority, + order_tag: Option, + data: ByteStreamReader, + sent: usize, +} + +impl SendQueue { + fn new() -> Self { + Self { + items: Vec::with_capacity(64), + } + } + fn push(&mut self, item: SendQueueItem) { + let prio = item.prio; + let pos_prio = match self.items.binary_search_by(|(p, _)| p.cmp(&prio)) { + Ok(i) => i, + Err(i) => { + self.items.insert(i, (prio, SendQueuePriority::new())); + i + } + }; + self.items[pos_prio].1.push(item); + } + fn remove(&mut self, id: RequestID) { + for (_, prioq) in self.items.iter_mut() { + prioq.remove(id); + } + self.items.retain(|(_prio, q)| !q.is_empty()); + } + fn is_empty(&self) -> bool { + self.items.iter().all(|(_k, v)| v.is_empty()) + } + + // this is like an async fn, but hand implemented + fn next_ready(&mut self) -> SendQueuePollNextReady<'_> { + SendQueuePollNextReady { queue: self } + } +} + +impl SendQueuePriority { + fn new() -> Self { + Self { + items: VecDeque::new(), + order: HashMap::new(), + } + } + fn push(&mut self, item: SendQueueItem) { + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_vec = self.order.entry(stream).or_default(); + let i = order_vec.iter().take_while(|o2| **o2 < order).count(); + order_vec.insert(i, order); + } + self.items.push_back(item); + } + fn remove(&mut self, id: RequestID) { + if let Some(i) = self.items.iter().position(|x| x.id == id) { + let item = self.items.remove(i).unwrap(); + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_vec = self.order.get_mut(&stream).unwrap(); + let j = order_vec.iter().position(|x| *x == order).unwrap(); + order_vec.remove(j).unwrap(); + if order_vec.is_empty() { + self.order.remove(&stream); + } + } + } + } + fn is_empty(&self) -> bool { + self.items.is_empty() + } + fn poll_next_ready(&mut self, ctx: &mut Context<'_>) -> Poll<(RequestID, DataFrame)> { + // in step 1: poll only streams that have sent 0 bytes, we want to send them in priority + // as they most likely represent small requests to be sent first + // in step 2: poll all streams + for step in 0..2 { + for (j, item) in self.items.iter_mut().enumerate() { + if let Some(OrderTag(stream, order)) = item.order_tag { + if order > *self.order.get(&stream).unwrap().front().unwrap() { + continue; + } + } + + if step == 0 && item.sent > 0 { + continue; + } + + let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize); + if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) { + let id = item.id; + let eos = item.data.eos(); + + let packet = bytes_or_err.map_err(|e| match e { + ReadExactError::Stream(err) => err, + _ => unreachable!(), + }); + + let is_err = packet.is_err(); + let data_frame = DataFrame::from_packet(packet, !eos); + item.sent += data_frame.data().len(); + + if eos || is_err { + // If item had an order tag, remove it from the corresponding ordering list + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_stream = self.order.get_mut(&stream).unwrap(); + assert_eq!(order_stream.pop_front(), Some(order)); + if order_stream.is_empty() { + self.order.remove(&stream); + } + } + // Remove item from sending queue + self.items.remove(j); + } else if step == 0 { + // Step 0 means that this stream had not sent any bytes yet. + // Now that it has, and it was not an EOS, we know that it is bigger + // than one chunk so move it at the end of the queue. + let item = self.items.remove(j).unwrap(); + self.items.push_back(item); + } + + return Poll::Ready((id, data_frame)); + } + } + } + + Poll::Pending + } + fn dump(&self, prio: u8) -> String { + self.items + .iter() + .map(|i| format!("[{} {} {:?} @{}]", prio, i.id, i.order_tag, i.sent)) + .collect::>() + .join(" ") + } +} + +struct SendQueuePollNextReady<'a> { + queue: &'a mut SendQueue, +} + +impl<'a> futures::Future for SendQueuePollNextReady<'a> { + type Output = (RequestID, DataFrame); + + fn poll(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + for (i, (_prio, items_at_prio)) in self.queue.items.iter_mut().enumerate() { + if let Poll::Ready(res) = items_at_prio.poll_next_ready(ctx) { + if items_at_prio.is_empty() { + self.queue.items.remove(i); + } + return Poll::Ready(res); + } + } + // If the queue is empty, this futures is eternally pending. + // This is ok because we use it in a select with another future + // that can interrupt it. + Poll::Pending + } +} + +enum DataFrame { + /// a fixed size buffer containing some data + a boolean indicating whether + /// there may be more data coming from this stream. Can be used for some + /// optimization. It's an error to set it to false if there is more data, but it is correct + /// (albeit sub-optimal) to set it to true if there is nothing coming after + Data(Bytes, bool), + /// An error code automatically signals the end of the stream + Error(Bytes), +} + +impl DataFrame { + fn from_packet(p: Packet, has_cont: bool) -> Self { + match p { + Ok(bytes) => { + assert!(bytes.len() <= MAX_CHUNK_LENGTH as usize); + Self::Data(bytes, has_cont) + } + Err(e) => { + let mut buf = BytesMut::new(); + buf.put_u8(io_errorkind_to_u8(e.kind())); + + let msg = format!("{}", e).into_bytes(); + if msg.len() > (MAX_CHUNK_LENGTH - 1) as usize { + buf.put(&msg[..(MAX_CHUNK_LENGTH - 1) as usize]); + } else { + buf.put(&msg[..]); + } + + Self::Error(buf.freeze()) + } + } + } + + fn header(&self) -> [u8; 2] { + let header_u16 = match self { + DataFrame::Data(data, false) => data.len() as u16, + DataFrame::Data(data, true) => data.len() as u16 | CHUNK_FLAG_HAS_CONTINUATION, + DataFrame::Error(msg) => msg.len() as u16 | CHUNK_FLAG_ERROR, + }; + ChunkLength::to_be_bytes(header_u16) + } + + fn data(&self) -> &[u8] { + match self { + DataFrame::Data(ref data, _) => &data[..], + DataFrame::Error(ref msg) => &msg[..], + } + } +} + +/// The SendLoop trait, which is implemented both by the client and the server +/// connection objects (ServerConna and ClientConn) adds a method `.send_loop()` +/// that takes a channel of messages to send and an asynchronous writer, +/// and sends messages from the channel to the async writer, putting them in a queue +/// before being sent and doing the round-robin sending strategy. +/// +/// The `.send_loop()` exits when the sending end of the channel is closed, +/// or if there is an error at any time writing to the async writer. +pub(crate) trait SendLoop: Sync { + async fn send_loop( + self: Arc, + msg_recv: mpsc::UnboundedReceiver, + mut write: BoxStreamWrite, + debug_name: String, + ) -> Result<(), Error> + where + W: AsyncWriteExt + Unpin + Send + Sync, + { + let mut sending = SendQueue::new(); + let mut msg_recv = Some(msg_recv); + while msg_recv.is_some() || !sending.is_empty() { + trace!( + "send_loop({}): queue = {:?}", + debug_name, + sending + .items + .iter() + .map(|(prio, i)| i.dump(*prio)) + .collect::>() + .join(" ; ") + ); + + let recv_fut = async { + if let Some(chan) = &mut msg_recv { + chan.recv().await + } else { + futures::future::pending().await + } + }; + let send_fut = sending.next_ready(); + + // recv_fut is cancellation-safe according to tokio doc, + // send_fut is cancellation-safe as implemented above? + tokio::select! { + biased; // always read incoming channel first if it has data + sth = recv_fut => { + match sth { + Some(SendItem::Stream(id, prio, order_tag, data)) => { + trace!("send_loop({}): add stream {} to send", debug_name, id); + sending.push(SendQueueItem { + id, + prio, + order_tag, + data: ByteStreamReader::new(data), + sent: 0, + }) + } + Some(SendItem::Cancel(id)) => { + trace!("send_loop({}): cancelling {}", debug_name, id); + sending.remove(id); + let header_id = RequestID::to_be_bytes(id); + write.write_all(&header_id[..]).await?; + write.write_all(&ChunkLength::to_be_bytes(CANCEL_REQUEST)).await?; + write.flush().await?; + } + None => { + msg_recv = None; + } + }; + } + (id, data) = send_fut => { + trace!( + "send_loop({}): id {}, send {} bytes, header_size {}", + debug_name, + id, + data.data().len(), + hex::encode(data.header()) + ); + + let header_id = RequestID::to_be_bytes(id); + write.write_all(&header_id[..]).await?; + + write.write_all(&data.header()).await?; + write.write_all(data.data()).await?; + write.flush().await?; + } + } + } + + let _ = write.goodbye().await; + Ok(()) + } +} diff --git a/src/net/server.rs b/src/net/server.rs new file mode 100644 index 00000000..fb6c6366 --- /dev/null +++ b/src/net/server.rs @@ -0,0 +1,220 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use arc_swap::ArcSwapOption; +use log::*; + +use futures::io::{AsyncReadExt, AsyncWriteExt}; +use kuska_handshake::async_std::{handshake_server, BoxStream}; +use tokio::net::TcpStream; +use tokio::select; +use tokio::sync::{mpsc, watch}; +use tokio_util::compat::*; + +#[cfg(feature = "telemetry")] +use opentelemetry::{ + trace::{FutureExt, Span, SpanKind, TraceContextExt, TraceId, Tracer}, + Context, KeyValue, +}; +#[cfg(feature = "telemetry")] +use opentelemetry_contrib::trace::propagator::binary::*; +#[cfg(feature = "telemetry")] +use rand::{thread_rng, Rng}; + +use crate::error::*; +use crate::message::*; +use crate::netapp::*; +use crate::recv::*; +use crate::send::*; +use crate::stream::*; +use crate::util::*; + +// The client and server connection structs (client.rs and server.rs) +// build upon the chunking mechanism which is exclusively contained +// in proto.rs. +// Here, we just care about sending big messages without size limit. +// The format of these messages is described below. +// Chunking happens independently. + +// Request message format (client -> server): +// - u8 priority +// - u8 path length +// - [u8; path length] path +// - [u8; *] data + +// Response message format (server -> client): +// - u8 response code +// - [u8; *] response + +pub(crate) struct ServerConn { + pub(crate) remote_addr: SocketAddr, + pub(crate) peer_id: NodeID, + + netapp: Arc, + + resp_send: ArcSwapOption>, + running_handlers: Mutex>>, +} + +impl ServerConn { + pub(crate) async fn run( + netapp: Arc, + socket: TcpStream, + must_exit: watch::Receiver, + ) -> Result<(), Error> { + let remote_addr = socket.peer_addr()?; + let mut socket = socket.compat(); + + // Do handshake to authenticate client + let handshake = handshake_server( + &mut socket, + netapp.netid.clone(), + netapp.id, + netapp.privkey.clone(), + ) + .await?; + let peer_id = handshake.peer_pk; + + debug!( + "Handshake complete (server) with {}@{}", + hex::encode(peer_id), + remote_addr + ); + + // Create BoxStream layer that encodes content + let (read, write) = socket.split(); + let (read, mut write) = + BoxStream::from_handshake(read, write, handshake, 0x8000).split_read_write(); + + // Before doing anything, send version tag, so that client + // can check and disconnect if version is wrong + write.write_all(&netapp.version_tag[..]).await?; + write.flush().await?; + + // Build and launch stuff that handles requests server-side + let (resp_send, resp_recv) = mpsc::unbounded_channel(); + + let conn = Arc::new(ServerConn { + netapp: netapp.clone(), + remote_addr, + peer_id, + resp_send: ArcSwapOption::new(Some(Arc::new(resp_send))), + running_handlers: Mutex::new(HashMap::new()), + }); + + netapp.connected_as_server(peer_id, conn.clone()); + + let debug_name = format!("SRV {}", hex::encode(&peer_id[..8])); + let debug_name_2 = debug_name.clone(); + + let conn2 = conn.clone(); + let recv_future = tokio::spawn(async move { + select! { + r = conn2.recv_loop(read, debug_name_2) => r, + _ = await_exit(must_exit) => Ok(()) + } + }); + let send_future = tokio::spawn(conn.clone().send_loop(resp_recv, write, debug_name)); + + recv_future.await.log_err("ServerConn recv_loop"); + conn.resp_send.store(None); + send_future.await.log_err("ServerConn send_loop"); + + netapp.disconnected_as_server(&peer_id, conn); + + Ok(()) + } + + async fn recv_handler_aux(self: &Arc, req_enc: ReqEnc) -> Result { + let path = String::from_utf8(req_enc.path.to_vec())?; + + let handler_opt = { + let endpoints = self.netapp.endpoints.read().unwrap(); + endpoints.get(&path).map(|e| e.clone_endpoint()) + }; + + if let Some(handler) = handler_opt { + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let tracer = opentelemetry::global::tracer("netapp"); + + let mut span = if !req_enc.telemetry_id.is_empty() { + let propagator = BinaryPropagator::new(); + let context = propagator.from_bytes(req_enc.telemetry_id.to_vec()); + let context = Context::new().with_remote_span_context(context); + tracer.span_builder(format!(">> RPC {}", path)) + .with_kind(SpanKind::Server) + .start_with_context(&tracer, &context) + } else { + let mut rng = thread_rng(); + let trace_id = TraceId::from_bytes(rng.gen()); + tracer + .span_builder(format!(">> RPC {}", path)) + .with_kind(SpanKind::Server) + .with_trace_id(trace_id) + .start(&tracer) + }; + span.set_attribute(KeyValue::new("path", path.to_string())); + span.set_attribute(KeyValue::new("len_query_msg", req_enc.msg.len() as i64)); + + handler.handle(req_enc, self.peer_id) + .with_context(Context::current_with_span(span)) + .await + } else { + handler.handle(req_enc, self.peer_id).await + } + } + } else { + Err(Error::NoHandler) + } + } +} + +impl SendLoop for ServerConn {} + +impl RecvLoop for ServerConn { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream) { + let resp_send = match self.resp_send.load_full() { + Some(c) => c, + None => return, + }; + + let mut rh = self.running_handlers.lock().unwrap(); + + let self2 = self.clone(); + let jh = tokio::spawn(async move { + debug!("server: recv_handler got {}", id); + + let (prio, resp_enc_result) = match ReqEnc::decode(stream).await { + Ok(req_enc) => (req_enc.prio, self2.recv_handler_aux(req_enc).await), + Err(e) => (PRIO_NORMAL, Err(e)), + }; + + debug!("server: sending response to {}", id); + + let (resp_stream, resp_order) = RespEnc::encode(resp_enc_result); + resp_send + .send(SendItem::Stream(id, prio, resp_order, resp_stream)) + .log_err("ServerConn recv_handler send resp bytes"); + + self2.running_handlers.lock().unwrap().remove(&id); + }); + + rh.insert(id, jh); + } + + fn cancel_handler(self: &Arc, id: RequestID) { + trace!("received cancel for request {}", id); + + // If the handler is still running, abort it now + if let Some(jh) = self.running_handlers.lock().unwrap().remove(&id) { + jh.abort(); + } + + // Inform the response sender that we don't need to send the response + if let Some(resp_send) = self.resp_send.load_full() { + let _ = resp_send.send(SendItem::Cancel(id)); + } + } +} diff --git a/src/net/stream.rs b/src/net/stream.rs new file mode 100644 index 00000000..c973f9a7 --- /dev/null +++ b/src/net/stream.rs @@ -0,0 +1,213 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use bytes::Bytes; + +use futures::Future; +use futures::{Stream, StreamExt}; +use tokio::io::AsyncRead; + +use crate::bytes_buf::BytesBuf; + +/// A stream of bytes (click to read more). +/// +/// When sent through Netapp, the Vec may be split in smaller chunk in such a way +/// consecutive Vec may get merged, but Vec and error code may not be reordered +/// +/// Items sent in the ByteStream may be errors of type `std::io::Error`. +/// An error indicates the end of the ByteStream: a reader should no longer read +/// after receiving an error, and a writer should stop writing after sending an error. +pub type ByteStream = Pin + Send + Sync>>; + +/// A packet sent in a ByteStream, which may contain either +/// a Bytes object or an error +pub type Packet = Result; + +// ---- + +/// A helper struct to read defined lengths of data from a BytesStream +pub struct ByteStreamReader { + stream: ByteStream, + buf: BytesBuf, + eos: bool, + err: Option, +} + +impl ByteStreamReader { + /// Creates a new `ByteStreamReader` from a `ByteStream` + pub fn new(stream: ByteStream) -> Self { + ByteStreamReader { + stream, + buf: BytesBuf::new(), + eos: false, + err: None, + } + } + + /// Read exactly `read_len` bytes from the underlying stream + /// (returns a future) + pub fn read_exact(&mut self, read_len: usize) -> ByteStreamReadExact<'_> { + ByteStreamReadExact { + reader: self, + read_len, + fail_on_eos: true, + } + } + + /// Read at most `read_len` bytes from the underlying stream, or less + /// if the end of the stream is reached (returns a future) + pub fn read_exact_or_eos(&mut self, read_len: usize) -> ByteStreamReadExact<'_> { + ByteStreamReadExact { + reader: self, + read_len, + fail_on_eos: false, + } + } + + /// Read exactly one byte from the underlying stream and returns it + /// as an u8 + pub async fn read_u8(&mut self) -> Result { + Ok(self.read_exact(1).await?[0]) + } + + /// Read exactly two bytes from the underlying stream and returns them as an u16 (using + /// big-endian decoding) + pub async fn read_u16(&mut self) -> Result { + let bytes = self.read_exact(2).await?; + let mut b = [0u8; 2]; + b.copy_from_slice(&bytes[..]); + Ok(u16::from_be_bytes(b)) + } + + /// Read exactly four bytes from the underlying stream and returns them as an u32 (using + /// big-endian decoding) + pub async fn read_u32(&mut self) -> Result { + let bytes = self.read_exact(4).await?; + let mut b = [0u8; 4]; + b.copy_from_slice(&bytes[..]); + Ok(u32::from_be_bytes(b)) + } + + /// Transforms the stream reader back into the underlying stream (starting + /// after everything that the reader has read) + pub fn into_stream(self) -> ByteStream { + let buf_stream = futures::stream::iter(self.buf.into_slices().into_iter().map(Ok)); + if let Some(err) = self.err { + Box::pin(buf_stream.chain(futures::stream::once(async move { Err(err) }))) + } else if self.eos { + Box::pin(buf_stream) + } else { + Box::pin(buf_stream.chain(self.stream)) + } + } + + /// Tries to fill the internal read buffer from the underlying stream if it is empty. + /// Calling this might be necessary to ensure that `.eos()` returns a correct + /// result, otherwise the reader might not be aware that the underlying + /// stream has nothing left to return. + pub async fn fill_buffer(&mut self) { + if self.buf.is_empty() { + let packet = self.stream.next().await; + self.add_stream_next(packet); + } + } + + /// Clears the internal read buffer and returns its content + pub fn take_buffer(&mut self) -> Bytes { + self.buf.take_all() + } + + /// Returns true if the end of the underlying stream has been reached + pub fn eos(&self) -> bool { + self.buf.is_empty() && self.eos + } + + fn try_get(&mut self, read_len: usize) -> Option { + self.buf.take_exact(read_len) + } + + fn add_stream_next(&mut self, packet: Option) { + match packet { + Some(Ok(slice)) => { + self.buf.extend(slice); + } + Some(Err(e)) => { + self.err = Some(e); + self.eos = true; + } + None => { + self.eos = true; + } + } + } +} + +/// The error kind that can be returned by `ByteStreamReader::read_exact` and +/// `ByteStreamReader::read_exact_or_eos` +pub enum ReadExactError { + /// The end of the stream was reached before the requested number of bytes could be read + UnexpectedEos, + /// The underlying data stream returned an IO error when trying to read + Stream(std::io::Error), +} + +/// The future returned by `ByteStreamReader::read_exact` and +/// `ByteStreamReader::read_exact_or_eos` +#[pin_project::pin_project] +pub struct ByteStreamReadExact<'a> { + #[pin] + reader: &'a mut ByteStreamReader, + read_len: usize, + fail_on_eos: bool, +} + +impl<'a> Future for ByteStreamReadExact<'a> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + loop { + if let Some(bytes) = this.reader.try_get(*this.read_len) { + return Poll::Ready(Ok(bytes)); + } + if let Some(err) = &this.reader.err { + let err = std::io::Error::new(err.kind(), format!("{}", err)); + return Poll::Ready(Err(ReadExactError::Stream(err))); + } + if this.reader.eos { + if *this.fail_on_eos { + return Poll::Ready(Err(ReadExactError::UnexpectedEos)); + } else { + return Poll::Ready(Ok(this.reader.take_buffer())); + } + } + + let next_packet = futures::ready!(this.reader.stream.as_mut().poll_next(cx)); + this.reader.add_stream_next(next_packet); + } + } +} + +// ---- + +/// Turns a `tokio::io::AsyncRead` asynchronous reader into a `ByteStream` +pub fn asyncread_stream(reader: R) -> ByteStream { + Box::pin(tokio_util::io::ReaderStream::new(reader)) +} + +/// Turns a `ByteStream` into a `tokio::io::AsyncRead` asynchronous reader +pub fn stream_asyncread(stream: ByteStream) -> impl AsyncRead + Send + Sync + 'static { + tokio_util::io::StreamReader::new(stream) +} + +/// Reads all of the content of a `ByteStream` into a BytesBuf +/// that contains everything +pub async fn read_stream_to_end(mut stream: ByteStream) -> Result { + let mut buf = BytesBuf::new(); + while let Some(part) = stream.next().await { + buf.extend(part?); + } + + Ok(buf) +} diff --git a/src/net/test.rs b/src/net/test.rs new file mode 100644 index 00000000..3cf446bd --- /dev/null +++ b/src/net/test.rs @@ -0,0 +1,118 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use tokio::select; +use tokio::sync::watch; + +use sodiumoxide::crypto::auth; +use sodiumoxide::crypto::sign::ed25519; + +use crate::netapp::*; +use crate::peering::*; +use crate::NodeID; + +#[tokio::test(flavor = "current_thread")] +async fn test_with_basic_scheduler() { + pretty_env_logger::init(); + run_test(19980).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_with_threaded_scheduler() { + run_test(19990).await +} + +async fn run_test(port_base: u16) { + select! { + _ = run_test_inner(port_base) => (), + _ = tokio::time::sleep(Duration::from_secs(20)) => panic!("timeout"), + } +} + +async fn run_test_inner(port_base: u16) { + let netid = auth::gen_key(); + + let (pk1, sk1) = ed25519::gen_keypair(); + let (pk2, sk2) = ed25519::gen_keypair(); + let (pk3, sk3) = ed25519::gen_keypair(); + + let addr1: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base); + let addr2: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base + 1); + let addr3: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base + 2); + + let (stop_tx, stop_rx) = watch::channel(false); + + let (thread1, _netapp1, peering1) = + run_netapp(netid.clone(), pk1, sk1, addr1, vec![], stop_rx.clone()); + tokio::time::sleep(Duration::from_secs(2)).await; + + // Connect second node and check it peers with everyone + let (thread2, _netapp2, peering2) = run_netapp( + netid.clone(), + pk2, + sk2, + addr2, + vec![(pk1, addr1)], + stop_rx.clone(), + ); + tokio::time::sleep(Duration::from_secs(3)).await; + + let pl1 = peering1.get_peer_list(); + println!("A pl1: {:?}", pl1); + assert_eq!(pl1.len(), 2); + + let pl2 = peering2.get_peer_list(); + println!("A pl2: {:?}", pl2); + assert_eq!(pl2.len(), 2); + + // Connect third node and check it peers with everyone + let (thread3, _netapp3, peering3) = + run_netapp(netid, pk3, sk3, addr3, vec![(pk2, addr2)], stop_rx.clone()); + tokio::time::sleep(Duration::from_secs(3)).await; + + let pl1 = peering1.get_peer_list(); + println!("B pl1: {:?}", pl1); + assert_eq!(pl1.len(), 3); + + let pl2 = peering2.get_peer_list(); + println!("B pl2: {:?}", pl2); + assert_eq!(pl2.len(), 3); + + let pl3 = peering3.get_peer_list(); + println!("B pl3: {:?}", pl3); + assert_eq!(pl3.len(), 3); + + // Send stop signal and wait for everyone to finish + stop_tx.send(true).unwrap(); + thread1.await.unwrap(); + thread2.await.unwrap(); + thread3.await.unwrap(); +} + +fn run_netapp( + netid: auth::Key, + _pk: NodeID, + sk: ed25519::SecretKey, + listen_addr: SocketAddr, + bootstrap_peers: Vec<(NodeID, SocketAddr)>, + must_exit: watch::Receiver, +) -> ( + tokio::task::JoinHandle<()>, + Arc, + Arc, +) { + let netapp = NetApp::new(0u64, netid, sk, None); + let peering = PeeringManager::new(netapp.clone(), bootstrap_peers, None); + + let peering2 = peering.clone(); + let netapp2 = netapp.clone(); + let fut = tokio::spawn(async move { + tokio::join!( + netapp2.listen(listen_addr, None, must_exit.clone()), + peering2.run(must_exit.clone()), + ); + }); + + (fut, netapp, peering) +} diff --git a/src/net/util.rs b/src/net/util.rs new file mode 100644 index 00000000..35a3be1e --- /dev/null +++ b/src/net/util.rs @@ -0,0 +1,96 @@ +use std::net::SocketAddr; + +use log::info; +use serde::Serialize; + +use tokio::sync::watch; + +use crate::netapp::*; + +/// Utility function: encodes any serializable value in MessagePack binary format +/// using the RMP library. +/// +/// Field names and variant names are included in the serialization. +/// This is used internally by the netapp communication protocol. +pub fn rmp_to_vec_all_named(val: &T) -> Result, rmp_serde::encode::Error> +where + T: Serialize + ?Sized, +{ + let mut wr = Vec::with_capacity(128); + let mut se = rmp_serde::Serializer::new(&mut wr).with_struct_map(); + val.serialize(&mut se)?; + Ok(wr) +} + +/// This async function returns only when a true signal was received +/// from a watcher that tells us when to exit. +/// +/// Useful in a select statement to interrupt another +/// future: +/// ```ignore +/// select!( +/// _ = a_long_task() => Success, +/// _ = await_exit(must_exit) => Interrupted, +/// ) +/// ``` +pub async fn await_exit(mut must_exit: watch::Receiver) { + while !*must_exit.borrow_and_update() { + if must_exit.changed().await.is_err() { + break; + } + } +} + +/// Creates a watch that contains `false`, and that changes +/// to `true` when a Ctrl+C signal is received. +pub fn watch_ctrl_c() -> watch::Receiver { + let (send_cancel, watch_cancel) = watch::channel(false); + tokio::spawn(async move { + tokio::signal::ctrl_c() + .await + .expect("failed to install CTRL+C signal handler"); + info!("Received CTRL+C, shutting down."); + send_cancel.send(true).unwrap(); + }); + watch_cancel +} + +/// Parse a peer's address including public key, written in the format: +/// `@:` +pub fn parse_peer_addr(peer: &str) -> Option<(NodeID, SocketAddr)> { + let delim = peer.find('@')?; + let (key, ip) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let ip = ip[1..].parse::().ok()?; + Some((pubkey, ip)) +} + +/// Parse and resolve a peer's address including public key, written in the format: +/// `@:` +pub fn parse_and_resolve_peer_addr(peer: &str) -> Option<(NodeID, Vec)> { + use std::net::ToSocketAddrs; + + let delim = peer.find('@')?; + let (key, host) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let hosts = host[1..].to_socket_addrs().ok()?.collect::>(); + if hosts.is_empty() { + return None; + } + Some((pubkey, hosts)) +} + +/// async version of parse_and_resolve_peer_addr +pub async fn parse_and_resolve_peer_addr_async(peer: &str) -> Option<(NodeID, Vec)> { + let delim = peer.find('@')?; + let (key, host) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let hosts = tokio::net::lookup_host(&host[1..]) + .await + .ok()? + .collect::>(); + if hosts.is_empty() { + return None; + } + Some((pubkey, hosts)) +} diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 2f22cd28..e23f4bca 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -15,43 +15,39 @@ path = "lib.rs" [dependencies] format_table.workspace = true -garage_db.workspace = true garage_util.workspace = true +garage_net.workspace = true -arc-swap = "1.0" -bytes = "1.0" -bytesize = "1.1" -gethostname = "0.4" -hex = "0.4" -tracing = "0.1" -rand = "0.8" -itertools="0.10" -sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -nix = { version = "0.27", default-features = false, features = ["fs"] } +arc-swap.workspace = true +bytesize.workspace = true +gethostname.workspace = true +hex.workspace = true +ipnet.workspace = true +tracing.workspace = true +rand.workspace = true +itertools.workspace = true +sodiumoxide.workspace = true +nix.workspace = true -async-trait = "0.1.7" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -serde_json = "1.0" -err-derive = { version = "0.3", optional = true } +async-trait.workspace = true +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +thiserror = { workspace = true, optional = true } # newer version requires rust edition 2021 -kube = { version = "0.75", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"], optional = true } -k8s-openapi = { version = "0.16", features = ["v1_22"], optional = true } -schemars = { version = "0.8", optional = true } -reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls-manual-roots", "json"] } +kube = { workspace = true, optional = true } +k8s-openapi = { workspace = true, optional = true } +schemars = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } -pnet_datalink = "0.33" +pnet_datalink.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-stream = { version = "0.1", features = ["net"] } -opentelemetry = "0.17" - -netapp = { version = "=0.5.2", features = ["telemetry"] } +futures.workspace = true +tokio.workspace = true +opentelemetry.workspace = true [features] kubernetes-discovery = [ "kube", "k8s-openapi", "schemars" ] -consul-discovery = [ "reqwest", "err-derive" ] +consul-discovery = [ "reqwest", "thiserror" ] system-libs = [ "sodiumoxide/use-pkg-config" ] diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index ab8d1112..760e9fcb 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -3,10 +3,10 @@ use std::fs::File; use std::io::Read; use std::net::{IpAddr, SocketAddr}; -use err_derive::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; -use netapp::NodeID; +use garage_net::NodeID; use garage_util::config::ConsulDiscoveryAPI; use garage_util::config::ConsulDiscoveryConfig; @@ -148,7 +148,7 @@ impl ConsulDiscovery { ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); } else { warn!( - "Could not process node spec from Consul: {:?} (invalid IP or public key)", + "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", ent ); } @@ -219,12 +219,12 @@ impl ConsulDiscovery { /// Regroup all Consul discovery errors #[derive(Debug, Error)] pub enum ConsulError { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] std::io::Error), - #[error(display = "HTTP error: {}", _0)] - Reqwest(#[error(source)] reqwest::Error), - #[error(display = "Invalid Consul TLS configuration")] + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("HTTP error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("Invalid Consul TLS configuration")] InvalidTLSConfig, - #[error(display = "Token error: {}", _0)] - Token(#[error(source)] reqwest::header::InvalidHeaderValue), + #[error("Token error: {0}")] + Token(#[from] reqwest::header::InvalidHeaderValue), } diff --git a/src/rpc/kubernetes.rs b/src/rpc/kubernetes.rs index 63c6567d..85254bb5 100644 --- a/src/rpc/kubernetes.rs +++ b/src/rpc/kubernetes.rs @@ -10,7 +10,7 @@ use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomRe use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use netapp::NodeID; +use garage_net::NodeID; use garage_util::config::KubernetesDiscoveryConfig; diff --git a/src/rpc/graph_algo.rs b/src/rpc/layout/graph_algo.rs similarity index 97% rename from src/rpc/graph_algo.rs rename to src/rpc/layout/graph_algo.rs index d8c6c9b9..29d4a043 100644 --- a/src/rpc/graph_algo.rs +++ b/src/rpc/layout/graph_algo.rs @@ -114,16 +114,6 @@ impl Graph { Ok(result) } - /// This function returns the value of the flow incoming to v. - pub fn get_inflow(&self, v: Vertex) -> Result { - let idv = self.get_vertex_id(&v)?; - let mut result = 0; - for edge in self.graph[idv].iter() { - result += max(0, self.graph[edge.dest][edge.rev].flow); - } - Ok(result) - } - /// This function returns the value of the flow outgoing from v. pub fn get_outflow(&self, v: Vertex) -> Result { let idv = self.get_vertex_id(&v)?; @@ -143,7 +133,7 @@ impl Graph { /// This function shuffles the order of the edge lists. It keeps the ids of the /// reversed edges consistent. fn shuffle_edges(&mut self) { - // We use deterministic randomness so that the layout calculation algorihtm + // We use deterministic randomness so that the layout calculation algorithm // will output the same thing every time it is run. This way, the results // pre-calculated in `garage layout show` will match exactly those used // in practice with `garage layout apply` diff --git a/src/rpc/layout/helper.rs b/src/rpc/layout/helper.rs new file mode 100644 index 00000000..c08a5629 --- /dev/null +++ b/src/rpc/layout/helper.rs @@ -0,0 +1,304 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use serde::{Deserialize, Serialize}; + +use garage_util::data::*; + +use super::*; +use crate::replication_mode::*; + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct RpcLayoutDigest { + /// Cluster layout version + pub current_version: u64, + /// Number of active layout versions + pub active_versions: usize, + /// Hash of cluster layout update trackers + pub trackers_hash: Hash, + /// Hash of cluster layout staging data + pub staging_hash: Hash, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct SyncLayoutDigest { + current: u64, + ack_map_min: u64, + min_stored: u64, +} + +pub struct LayoutHelper { + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + layout: Option, + + // cached values + ack_map_min: u64, + sync_map_min: u64, + + all_nodes: Vec, + all_nongateway_nodes: Vec, + + trackers_hash: Hash, + staging_hash: Hash, + is_check_ok: bool, + + // ack lock: counts in-progress write operations for each + // layout version ; we don't increase the ack update tracker + // while this lock is nonzero + pub(crate) ack_lock: HashMap, +} + +impl LayoutHelper { + pub fn new( + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + mut layout: LayoutHistory, + mut ack_lock: HashMap, + ) -> Self { + // In the new() function of the helper, we do a bunch of cleanup + // and calculations on the layout history to make sure things are + // correct and we have rapid access to important values such as + // the layout versions to use when reading to ensure consistency. + + if consistency_mode != ConsistencyMode::Consistent { + // Fast path for when no consistency is required. + // In this case we only need to keep the last version of the layout, + // we don't care about coordinating stuff in the cluster. + layout.keep_current_version_only(); + } + + layout.cleanup_old_versions(); + + let all_nodes = layout.get_all_nodes(); + let all_nongateway_nodes = layout.get_all_nongateway_nodes(); + + layout.clamp_update_trackers(&all_nodes); + + let min_version = layout.min_stored(); + + // ack_map_min is the minimum value of ack_map among all nodes + // in the cluster (gateway, non-gateway, current and previous layouts). + // It is the highest layout version which all of these nodes have + // acknowledged, indicating that they are aware of it and are no + // longer processing write operations that did not take it into account. + let ack_map_min = layout + .update_trackers + .ack_map + .min_among(&all_nodes, min_version); + + // sync_map_min is the minimum value of sync_map among storage nodes + // in the cluster (non-gateway nodes only, current and previous layouts). + // It is the highest layout version for which we know that all relevant + // storage nodes have fulfilled a sync, and therefore it is safe to + // use a read quorum within that layout to ensure consistency. + // Gateway nodes are excluded here because they hold no relevant data + // (they store the bucket and access key tables, but we don't have + // consistency on those). + // This value is calculated using quorums to allow progress even + // if not all nodes have successfully completed a sync. + let sync_map_min = + layout.calculate_sync_map_min_with_quorum(replication_factor, &all_nongateway_nodes); + + let trackers_hash = layout.calculate_trackers_hash(); + let staging_hash = layout.calculate_staging_hash(); + + ack_lock.retain(|_, cnt| *cnt.get_mut() > 0); + ack_lock + .entry(layout.current().version) + .or_insert(AtomicUsize::new(0)); + + let is_check_ok = layout.check().is_ok(); + + LayoutHelper { + replication_factor, + consistency_mode, + layout: Some(layout), + ack_map_min, + sync_map_min, + all_nodes, + all_nongateway_nodes, + trackers_hash, + staging_hash, + ack_lock, + is_check_ok, + } + } + + // ------------------ single updating function -------------- + + pub(crate) fn update(&mut self, f: F) -> bool + where + F: FnOnce(&mut LayoutHistory) -> bool, + { + let changed = f(self.layout.as_mut().unwrap()); + if changed { + *self = Self::new( + self.replication_factor, + self.consistency_mode, + self.layout.take().unwrap(), + std::mem::take(&mut self.ack_lock), + ); + } + changed + } + + // ------------------ read helpers --------------- + + pub fn inner(&self) -> &LayoutHistory { + self.layout.as_ref().unwrap() + } + + pub fn current(&self) -> &LayoutVersion { + self.inner().current() + } + + pub fn versions(&self) -> &[LayoutVersion] { + &self.inner().versions + } + + pub fn is_check_ok(&self) -> bool { + self.is_check_ok + } + + /// Return all nodes that have a role (gateway or storage) + /// in one of the currently active layout versions + pub fn all_nodes(&self) -> &[Uuid] { + &self.all_nodes + } + + /// Return all nodes that are configured to store data + /// in one of the currently active layout versions + pub fn all_nongateway_nodes(&self) -> &[Uuid] { + &self.all_nongateway_nodes + } + + pub fn ack_map_min(&self) -> u64 { + self.ack_map_min + } + + pub fn sync_map_min(&self) -> u64 { + self.sync_map_min + } + + pub fn sync_digest(&self) -> SyncLayoutDigest { + SyncLayoutDigest { + current: self.current().version, + ack_map_min: self.ack_map_min(), + min_stored: self.inner().min_stored(), + } + } + + pub fn read_nodes_of(&self, position: &Hash) -> Vec { + let sync_min = self.sync_map_min; + let version = self + .versions() + .iter() + .find(|x| x.version == sync_min) + .or(self.versions().last()) + .unwrap(); + version + .nodes_of(position, version.replication_factor) + .collect() + } + + pub fn storage_sets_of(&self, position: &Hash) -> Vec> { + self.versions() + .iter() + .map(|x| x.nodes_of(position, x.replication_factor).collect()) + .collect() + } + + pub fn storage_nodes_of(&self, position: &Hash) -> Vec { + let mut ret = vec![]; + for version in self.versions().iter() { + ret.extend(version.nodes_of(position, version.replication_factor)); + } + ret.sort(); + ret.dedup(); + ret + } + + pub fn current_storage_nodes_of(&self, position: &Hash) -> Vec { + let ver = self.current(); + ver.nodes_of(position, ver.replication_factor).collect() + } + + pub fn trackers_hash(&self) -> Hash { + self.trackers_hash + } + + pub fn staging_hash(&self) -> Hash { + self.staging_hash + } + + pub fn digest(&self) -> RpcLayoutDigest { + RpcLayoutDigest { + current_version: self.current().version, + active_versions: self.versions().len(), + trackers_hash: self.trackers_hash, + staging_hash: self.staging_hash, + } + } + + // ------------------ helpers for update tracking --------------- + + pub(crate) fn update_update_trackers(&mut self, local_node_id: Uuid) { + // Ensure trackers for this node's values are up-to-date + + // 1. Acknowledge the last layout version which is not currently + // locked by an in-progress write operation + self.update_ack_to_max_free(local_node_id); + + // 2. Assume the data on this node is sync'ed up at least to + // the first layout version in the history + let first_version = self.inner().min_stored(); + self.update(|layout| { + layout + .update_trackers + .sync_map + .set_max(local_node_id, first_version) + }); + + // 3. Acknowledge everyone has synced up to min(self.sync_map) + let sync_map_min = self.sync_map_min; + self.update(|layout| { + layout + .update_trackers + .sync_ack_map + .set_max(local_node_id, sync_map_min) + }); + + debug!("ack_map: {:?}", self.inner().update_trackers.ack_map); + debug!("sync_map: {:?}", self.inner().update_trackers.sync_map); + debug!( + "sync_ack_map: {:?}", + self.inner().update_trackers.sync_ack_map + ); + } + + pub(crate) fn update_ack_to_max_free(&mut self, local_node_id: Uuid) -> bool { + let max_free = self + .versions() + .iter() + .map(|x| x.version) + .skip_while(|v| { + self.ack_lock + .get(v) + .map(|x| x.load(Ordering::Relaxed) == 0) + .unwrap_or(true) + }) + .next() + .unwrap_or(self.current().version); + let changed = self.update(|layout| { + layout + .update_trackers + .ack_map + .set_max(local_node_id, max_free) + }); + if changed { + info!("ack_until updated to {}", max_free); + } + changed + } +} diff --git a/src/rpc/layout/history.rs b/src/rpc/layout/history.rs new file mode 100644 index 00000000..574c50c2 --- /dev/null +++ b/src/rpc/layout/history.rs @@ -0,0 +1,317 @@ +use std::collections::HashSet; + +use garage_util::crdt::{Crdt, Lww, LwwMap}; +use garage_util::data::*; +use garage_util::encode::nonversioned_encode; +use garage_util::error::*; + +use super::*; +use crate::replication_mode::*; + +impl LayoutHistory { + pub fn new(replication_factor: ReplicationFactor) -> Self { + let version = LayoutVersion::new(replication_factor.into()); + + let staging = LayoutStaging { + parameters: Lww::::new(version.parameters), + roles: LwwMap::new(), + }; + + LayoutHistory { + versions: vec![version], + old_versions: vec![], + update_trackers: Default::default(), + staging: Lww::raw(0, staging), + } + } + + // ------------------ who stores what now? --------------- + + /// Returns the layout version with the highest number + pub fn current(&self) -> &LayoutVersion { + self.versions.last().as_ref().unwrap() + } + + /// Returns the version number of the oldest layout version still active + pub fn min_stored(&self) -> u64 { + self.versions.first().as_ref().unwrap().version + } + + /// Calculate the set of all nodes that have a role (gateway or storage) + /// in one of the currently active layout versions + pub fn get_all_nodes(&self) -> Vec { + if self.versions.len() == 1 { + self.versions[0].all_nodes().to_vec() + } else { + let set = self + .versions + .iter() + .flat_map(|x| x.all_nodes()) + .collect::>(); + set.into_iter().copied().collect::>() + } + } + + /// Calculate the set of all nodes that are configured to store data + /// in one of the currently active layout versions + pub(crate) fn get_all_nongateway_nodes(&self) -> Vec { + if self.versions.len() == 1 { + self.versions[0].nongateway_nodes().to_vec() + } else { + let set = self + .versions + .iter() + .flat_map(|x| x.nongateway_nodes()) + .collect::>(); + set.into_iter().copied().collect::>() + } + } + + // ---- housekeeping (all invoked by LayoutHelper) ---- + + pub(crate) fn keep_current_version_only(&mut self) { + while self.versions.len() > 1 { + let removed = self.versions.remove(0); + self.old_versions.push(removed); + } + } + + pub(crate) fn cleanup_old_versions(&mut self) { + // If there are invalid versions before valid versions, remove them + if self.versions.len() > 1 && self.current().check().is_ok() { + while self.versions.len() > 1 && self.versions.first().unwrap().check().is_err() { + let removed = self.versions.remove(0); + info!( + "Layout history: pruning old invalid version {}", + removed.version + ); + } + } + + // If there are old versions that no one is reading from anymore, + // remove them (keep them in self.old_versions). + // ASSUMPTION: we only care about where nodes in the current layout version + // are reading from, as we assume older nodes are being discarded. + let current_nodes = &self.current().node_id_vec; + let min_version = self.min_stored(); + let sync_ack_map_min = self + .update_trackers + .sync_ack_map + .min_among(current_nodes, min_version); + while self.min_stored() < sync_ack_map_min { + assert!(self.versions.len() > 1); + let removed = self.versions.remove(0); + info!( + "Layout history: moving version {} to old_versions", + removed.version + ); + self.old_versions.push(removed); + } + + while self.old_versions.len() > OLD_VERSION_COUNT { + let removed = self.old_versions.remove(0); + info!("Layout history: removing old_version {}", removed.version); + } + } + + pub(crate) fn clamp_update_trackers(&mut self, nodes: &[Uuid]) { + let min_v = self.min_stored(); + for node in nodes { + self.update_trackers.ack_map.set_max(*node, min_v); + self.update_trackers.sync_map.set_max(*node, min_v); + self.update_trackers.sync_ack_map.set_max(*node, min_v); + } + } + + pub(crate) fn calculate_sync_map_min_with_quorum( + &self, + replication_factor: ReplicationFactor, + all_nongateway_nodes: &[Uuid], + ) -> u64 { + // This function calculates the minimum layout version from which + // it is safe to read if we want to maintain read-after-write consistency. + // In the general case the computation can be a bit expensive so + // we try to optimize it in several ways. + + // If there is only one layout version, we know that's the one + // we need to read from. + if self.versions.len() == 1 { + return self.current().version; + } + + let quorum = replication_factor.write_quorum(ConsistencyMode::Consistent); + + let min_version = self.min_stored(); + let global_min = self + .update_trackers + .sync_map + .min_among(all_nongateway_nodes, min_version); + + // If the write quorums are equal to the total number of nodes, + // i.e. no writes can succeed while they are not written to all nodes, + // then we must in all case wait for all nodes to complete a sync. + // This is represented by reading from the layout with version + // number global_min, the smallest layout version for which all nodes + // have completed a sync. + if quorum == self.current().replication_factor { + return global_min; + } + + // In the general case, we need to look at all write sets for all partitions, + // and find a safe layout version to read for that partition. We then + // take the minimum value among all partition as the safe layout version + // to read in all cases (the layout version to which all reads are directed). + let mut current_min = self.current().version; + let mut sets_done = HashSet::>::new(); + + for (_, p_hash) in self.current().partitions() { + for v in self.versions.iter() { + if v.version == self.current().version { + // We don't care about whether nodes in the latest layout version + // have completed a sync or not, as the sync is push-only + // and by definition nodes in the latest layout version do not + // hold data that must be pushed to nodes in the latest layout + // version, since that's the same version (any data that's + // already in the latest version is assumed to have been written + // by an operation that ensured a quorum of writes within + // that version). + continue; + } + + // Determine set of nodes for partition p in layout version v. + // Sort the node set to avoid duplicate computations. + let mut set = v + .nodes_of(&p_hash, v.replication_factor) + .collect::>(); + set.sort(); + + // If this set was already processed, skip it. + if sets_done.contains(&set) { + continue; + } + + // Find the value of the sync update trackers that is the + // highest possible minimum within a quorum of nodes. + let mut sync_values = set + .iter() + .map(|x| self.update_trackers.sync_map.get(x, min_version)) + .collect::>(); + sync_values.sort(); + let set_min = sync_values[sync_values.len() - quorum]; + if set_min < current_min { + current_min = set_min; + } + // defavorable case, we know we are at the smallest possible version, + // so we can stop early + assert!(current_min >= global_min); + if current_min == global_min { + return current_min; + } + + // Add set to already processed sets + sets_done.insert(set); + } + } + + current_min + } + + pub(crate) fn calculate_trackers_hash(&self) -> Hash { + blake2sum(&nonversioned_encode(&self.update_trackers).unwrap()[..]) + } + + pub(crate) fn calculate_staging_hash(&self) -> Hash { + blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]) + } + + // ================== updates to layout, public interface =================== + + pub fn merge(&mut self, other: &LayoutHistory) -> bool { + // If our current layout version is completely out-of-date, + // forget everything we know and replace it by incoming layout data. + if self.current().version < other.min_stored() { + *self = other.clone(); + return true; + } + + let mut changed = false; + + // Add any new versions to history + for v2 in other.versions.iter() { + if v2.version == self.current().version + 1 { + // This is the next version, add it to our version list + self.versions.push(v2.clone()); + changed = true; + } else if let Some(v1) = self.versions.iter().find(|v| v.version == v2.version) { + // Version is already present, check consistency + if v1 != v2 { + error!("Inconsistent layout histories: different layout compositions for version {}. Your cluster will be broken as long as this layout version is not replaced.", v2.version); + } + } else { + // This is an older version + assert!(v2.version < self.min_stored()); + } + } + + // Merge trackers + let c = self.update_trackers.merge(&other.update_trackers); + changed = changed || c; + + // Merge staged layout changes + if self.staging != other.staging { + let prev_staging = self.staging.clone(); + self.staging.merge(&other.staging); + changed = changed || self.staging != prev_staging; + } + + changed + } + + pub fn apply_staged_changes(mut self, version: Option) -> Result<(Self, Message), Error> { + match version { + None => { + let error = r#" +Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. +To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. + "#; + return Err(Error::Message(error.into())); + } + Some(v) => { + if v != self.current().version + 1 { + return Err(Error::Message("Invalid new layout version".into())); + } + } + } + + // Compute new version and add it to history + let (new_version, msg) = self + .current() + .clone() + .calculate_next_version(self.staging.get())?; + + self.versions.push(new_version); + self.cleanup_old_versions(); + + // Reset the staged layout changes + self.staging.update(LayoutStaging { + parameters: self.staging.get().parameters.clone(), + roles: LwwMap::new(), + }); + + Ok((self, msg)) + } + + pub fn revert_staged_changes(mut self) -> Result { + self.staging.update(LayoutStaging { + parameters: Lww::new(self.current().parameters), + roles: LwwMap::new(), + }); + + Ok(self) + } + + pub fn check(&self) -> Result<(), String> { + // TODO: anything more ? + self.current().check() + } +} diff --git a/src/rpc/layout/manager.rs b/src/rpc/layout/manager.rs new file mode 100644 index 00000000..bb8000bd --- /dev/null +++ b/src/rpc/layout/manager.rs @@ -0,0 +1,379 @@ +use std::collections::HashMap; +use std::sync::{atomic::Ordering, Arc, Mutex, RwLock, RwLockReadGuard}; +use std::time::Duration; + +use tokio::sync::Notify; + +use garage_net::endpoint::Endpoint; +use garage_net::peering::PeeringManager; +use garage_net::NodeID; + +use garage_util::config::Config; +use garage_util::data::*; +use garage_util::error::*; +use garage_util::persister::Persister; + +use super::*; +use crate::replication_mode::*; +use crate::rpc_helper::*; +use crate::system::*; + +pub struct LayoutManager { + node_id: Uuid, + replication_factor: ReplicationFactor, + persist_cluster_layout: Persister, + + layout: Arc>, + pub(crate) change_notify: Arc, + + table_sync_version: Mutex>, + + pub(crate) rpc_helper: RpcHelper, + system_endpoint: Arc>, +} + +impl LayoutManager { + pub fn new( + config: &Config, + node_id: NodeID, + system_endpoint: Arc>, + peering: Arc, + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + ) -> Result, Error> { + let persist_cluster_layout: Persister = + Persister::new(&config.metadata_dir, "cluster_layout"); + + let cluster_layout = match persist_cluster_layout.load() { + Ok(x) => { + if x.current().replication_factor != replication_factor.replication_factor() { + return Err(Error::Message(format!( + "Previous cluster layout has replication factor {}, which is different than the one specified in the config file ({}). The previous cluster layout can be purged, if you know what you are doing, simply by deleting the `cluster_layout` file in your metadata directory.", + x.current().replication_factor, + replication_factor.replication_factor() + ))); + } + x + } + Err(e) => { + info!( + "No valid previous cluster layout stored ({}), starting fresh.", + e + ); + LayoutHistory::new(replication_factor) + } + }; + + let mut cluster_layout = LayoutHelper::new( + replication_factor, + consistency_mode, + cluster_layout, + Default::default(), + ); + cluster_layout.update_update_trackers(node_id.into()); + + let layout = Arc::new(RwLock::new(cluster_layout)); + let change_notify = Arc::new(Notify::new()); + + let rpc_helper = RpcHelper::new( + node_id.into(), + peering, + layout.clone(), + config.rpc_timeout_msec.map(Duration::from_millis), + ); + + Ok(Arc::new(Self { + node_id: node_id.into(), + replication_factor, + persist_cluster_layout, + layout, + change_notify, + table_sync_version: Mutex::new(HashMap::new()), + system_endpoint, + rpc_helper, + })) + } + + // ---- PUBLIC INTERFACE ---- + + pub fn layout(&self) -> RwLockReadGuard<'_, LayoutHelper> { + self.layout.read().unwrap() + } + + pub async fn update_cluster_layout( + self: &Arc, + layout: &LayoutHistory, + ) -> Result<(), Error> { + self.handle_advertise_cluster_layout(layout).await?; + Ok(()) + } + + pub fn add_table(&self, table_name: &'static str) { + let first_version = self.layout().versions().first().unwrap().version; + + self.table_sync_version + .lock() + .unwrap() + .insert(table_name.to_string(), first_version); + } + + pub fn sync_table_until(self: &Arc, table_name: &'static str, version: u64) { + let mut table_sync_version = self.table_sync_version.lock().unwrap(); + *table_sync_version.get_mut(table_name).unwrap() = version; + let sync_until = table_sync_version.iter().map(|(_, v)| *v).min().unwrap(); + drop(table_sync_version); + + let mut layout = self.layout.write().unwrap(); + if layout.update(|l| l.update_trackers.sync_map.set_max(self.node_id, sync_until)) { + info!("sync_until updated to {}", sync_until); + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers( + layout.inner().update_trackers.clone(), + )); + } + } + + fn ack_new_version(self: &Arc) { + let mut layout = self.layout.write().unwrap(); + if layout.update_ack_to_max_free(self.node_id) { + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers( + layout.inner().update_trackers.clone(), + )); + } + } + + // ---- ACK LOCKING ---- + + pub fn write_sets_of(self: &Arc, position: &Hash) -> WriteLock>> { + let layout = self.layout(); + let version = layout.current().version; + let nodes = layout.storage_sets_of(position); + layout + .ack_lock + .get(&version) + .unwrap() + .fetch_add(1, Ordering::Relaxed); + WriteLock::new(version, self, nodes) + } + + // ---- INTERNALS --- + + fn merge_layout(&self, adv: &LayoutHistory) -> Option { + let mut layout = self.layout.write().unwrap(); + let prev_digest = layout.digest(); + let prev_layout_check = layout.is_check_ok(); + + if !prev_layout_check || adv.check().is_ok() { + if layout.update(|l| l.merge(adv)) { + layout.update_update_trackers(self.node_id); + if prev_layout_check && !layout.is_check_ok() { + panic!("Merged two correct layouts and got an incorrect layout."); + } + assert!(layout.digest() != prev_digest); + return Some(layout.inner().clone()); + } + } + + None + } + + fn merge_layout_trackers(&self, adv: &UpdateTrackers) -> Option { + let mut layout = self.layout.write().unwrap(); + let prev_digest = layout.digest(); + + if layout.inner().update_trackers != *adv { + if layout.update(|l| l.update_trackers.merge(adv)) { + layout.update_update_trackers(self.node_id); + assert!(layout.digest() != prev_digest); + return Some(layout.inner().update_trackers.clone()); + } + } + + None + } + + async fn pull_cluster_layout(self: &Arc, peer: Uuid) { + let resp = self + .rpc_helper + .call( + &self.system_endpoint, + peer, + SystemRpc::PullClusterLayout, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await; + if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp { + if let Err(e) = self.handle_advertise_cluster_layout(&layout).await { + warn!("In pull_cluster_layout: {}", e); + } + } + } + + async fn pull_cluster_layout_trackers(self: &Arc, peer: Uuid) { + let resp = self + .rpc_helper + .call( + &self.system_endpoint, + peer, + SystemRpc::PullClusterLayoutTrackers, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await; + if let Ok(SystemRpc::AdvertiseClusterLayoutTrackers(trackers)) = resp { + if let Err(e) = self + .handle_advertise_cluster_layout_trackers(&trackers) + .await + { + warn!("In pull_cluster_layout_trackers: {}", e); + } + } + } + + /// Save cluster layout data to disk + async fn save_cluster_layout(&self) { + let layout = self.layout.read().unwrap().inner().clone(); + if let Err(e) = self.persist_cluster_layout.save_async(&layout).await { + error!("Failed to save cluster_layout: {}", e); + } + } + + fn broadcast_update(self: &Arc, rpc: SystemRpc) { + tokio::spawn({ + let this = self.clone(); + async move { + if let Err(e) = this + .rpc_helper + .broadcast( + &this.system_endpoint, + rpc, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await + { + warn!("Error while broadcasting new cluster layout: {}", e); + } + } + }); + } + + // ---- RPC HANDLERS ---- + + pub(crate) fn handle_advertise_status(self: &Arc, from: Uuid, remote: &RpcLayoutDigest) { + let local = self.layout().digest(); + if remote.current_version > local.current_version + || remote.active_versions != local.active_versions + || remote.staging_hash != local.staging_hash + { + tokio::spawn({ + let this = self.clone(); + async move { this.pull_cluster_layout(from).await } + }); + } else if remote.trackers_hash != local.trackers_hash { + tokio::spawn({ + let this = self.clone(); + async move { this.pull_cluster_layout_trackers(from).await } + }); + } + } + + pub(crate) fn handle_pull_cluster_layout(&self) -> SystemRpc { + let layout = self.layout.read().unwrap().inner().clone(); + SystemRpc::AdvertiseClusterLayout(layout) + } + + pub(crate) fn handle_pull_cluster_layout_trackers(&self) -> SystemRpc { + let layout = self.layout.read().unwrap(); + SystemRpc::AdvertiseClusterLayoutTrackers(layout.inner().update_trackers.clone()) + } + + pub(crate) async fn handle_advertise_cluster_layout( + self: &Arc, + adv: &LayoutHistory, + ) -> Result { + debug!( + "handle_advertise_cluster_layout: {} versions, last={}, trackers={:?}", + adv.versions.len(), + adv.current().version, + adv.update_trackers + ); + + if adv.current().replication_factor != self.replication_factor.replication_factor() { + let msg = format!( + "Received a cluster layout from another node with replication factor {}, which is different from what we have in our configuration ({}). Discarding the cluster layout we received.", + adv.current().replication_factor, + self.replication_factor.replication_factor() + ); + error!("{}", msg); + return Err(Error::Message(msg)); + } + + if let Some(new_layout) = self.merge_layout(adv) { + debug!("handle_advertise_cluster_layout: some changes were added to the current stuff"); + + self.change_notify.notify_waiters(); + self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout)); + self.save_cluster_layout().await; + } + + Ok(SystemRpc::Ok) + } + + pub(crate) async fn handle_advertise_cluster_layout_trackers( + self: &Arc, + trackers: &UpdateTrackers, + ) -> Result { + debug!("handle_advertise_cluster_layout_trackers: {:?}", trackers); + + if let Some(new_trackers) = self.merge_layout_trackers(trackers) { + self.change_notify.notify_waiters(); + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers)); + self.save_cluster_layout().await; + } + + Ok(SystemRpc::Ok) + } +} + +// ---- ack lock ---- + +pub struct WriteLock { + layout_version: u64, + layout_manager: Arc, + value: T, +} + +impl WriteLock { + fn new(version: u64, layout_manager: &Arc, value: T) -> Self { + Self { + layout_version: version, + layout_manager: layout_manager.clone(), + value, + } + } +} + +impl AsRef for WriteLock { + fn as_ref(&self) -> &T { + &self.value + } +} + +impl AsMut for WriteLock { + fn as_mut(&mut self) -> &mut T { + &mut self.value + } +} + +impl Drop for WriteLock { + fn drop(&mut self) { + let layout = self.layout_manager.layout(); // acquire read lock + if let Some(counter) = layout.ack_lock.get(&self.layout_version) { + let prev_lock = counter.fetch_sub(1, Ordering::Relaxed); + if prev_lock == 1 && layout.current().version > self.layout_version { + drop(layout); // release read lock, write lock will be acquired + self.layout_manager.ack_new_version(); + } + } else { + error!("Could not find ack lock counter for layout version {}. This probably indicates a bug in Garage.", self.layout_version); + } + } +} diff --git a/src/rpc/layout/mod.rs b/src/rpc/layout/mod.rs new file mode 100644 index 00000000..ce21a524 --- /dev/null +++ b/src/rpc/layout/mod.rs @@ -0,0 +1,478 @@ +use std::fmt; + +use bytesize::ByteSize; + +use garage_util::crdt::{AutoCrdt, Crdt}; +use garage_util::data::Uuid; + +mod graph_algo; +mod helper; +mod history; +mod version; + +#[cfg(test)] +mod test; + +pub mod manager; + +// ---- re-exports ---- + +pub use helper::{LayoutHelper, RpcLayoutDigest, SyncLayoutDigest}; +pub use manager::WriteLock; +pub use version::*; + +// ---- defines: partitions ---- + +/// A partition id, which is stored on 16 bits +/// i.e. we have up to 2**16 partitions. +/// (in practice we have exactly 2**PARTITION_BITS partitions) +pub type Partition = u16; + +// TODO: make this constant parametrizable in the config file +// For deployments with many nodes it might make sense to bump +// it up to 10. +// Maximum value : 16 +/// How many bits from the hash are used to make partitions. Higher numbers means more fairness in +/// presence of numerous nodes, but exponentially bigger ring. Max 16 +pub const PARTITION_BITS: usize = 8; + +const NB_PARTITIONS: usize = 1usize << PARTITION_BITS; + +// ---- defines: nodes ---- + +// Type to store compactly the id of a node in the system +// Change this to u16 the day we want to have more than 256 nodes in a cluster +pub type CompactNodeType = u8; +pub const MAX_NODE_NUMBER: usize = 256; + +// ======== actual data structures for the layout data ======== +// ======== that is persisted to disk ======== +// some small utility impls are at the end of this file, +// but most of the code that actually computes stuff is in +// version.rs, history.rs and helper.rs + +mod v08 { + use crate::layout::CompactNodeType; + use garage_util::crdt::LwwMap; + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + + /// The layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct ClusterLayout { + pub version: u64, + + pub replication_factor: usize, + pub roles: LwwMap, + + // see comments in v010::ClusterLayout + pub node_id_vec: Vec, + #[serde(with = "serde_bytes")] + pub ring_assignation_data: Vec, + + /// Role changes which are staged for the next version of the layout + pub staging: LwwMap, + pub staging_hash: Hash, + } + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct NodeRoleV(pub Option); + + /// The user-assigned roles of cluster nodes + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct NodeRole { + /// Datacenter at which this entry belong. This information is used to + /// perform a better geodistribution + pub zone: String, + /// The capacity of the node + /// If this is set to None, the node does not participate in storing data for the system + /// and is only active as an API gateway to other nodes + pub capacity: Option, + /// A set of tags to recognize the node + pub tags: Vec, + } + + impl garage_util::migrate::InitialFormat for ClusterLayout {} +} + +mod v09 { + use super::v08; + use crate::layout::CompactNodeType; + use garage_util::crdt::{Lww, LwwMap}; + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + pub use v08::{NodeRole, NodeRoleV}; + + /// The layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct ClusterLayout { + pub version: u64, + + pub replication_factor: usize, + + /// This attribute is only used to retain the previously computed partition size, + /// to know to what extent does it change with the layout update. + pub partition_size: u64, + /// Parameters used to compute the assignment currently given by + /// ring_assignment_data + pub parameters: LayoutParameters, + + pub roles: LwwMap, + + // see comments in v010::ClusterLayout + pub node_id_vec: Vec, + #[serde(with = "serde_bytes")] + pub ring_assignment_data: Vec, + + /// Parameters to be used in the next partition assignment computation. + pub staging_parameters: Lww, + /// Role changes which are staged for the next version of the layout + pub staging_roles: LwwMap, + pub staging_hash: Hash, + } + + /// This struct is used to set the parameters to be used in the assignment computation + /// algorithm. It is stored as a Crdt. + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct LayoutParameters { + pub zone_redundancy: ZoneRedundancy, + } + + /// Zone redundancy: if set to AtLeast(x), the layout calculation will aim to store copies + /// of each partition on at least that number of different zones. + /// Otherwise, copies will be stored on the maximum possible number of zones. + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ZoneRedundancy { + AtLeast(usize), + Maximum, + } + + impl garage_util::migrate::Migrate for ClusterLayout { + const VERSION_MARKER: &'static [u8] = b"G09layout"; + + type Previous = v08::ClusterLayout; + + fn migrate(previous: Self::Previous) -> Self { + use itertools::Itertools; + + // In the old layout, capacities are in an arbitrary unit, + // but in the new layout they are in bytes. + // Here we arbitrarily multiply everything by 1G, + // such that 1 old capacity unit = 1GB in the new units. + // This is totally arbitrary and won't work for most users. + let cap_mul = 1024 * 1024 * 1024; + let roles = multiply_all_capacities(previous.roles, cap_mul); + let staging_roles = multiply_all_capacities(previous.staging, cap_mul); + let node_id_vec = previous.node_id_vec; + + // Determine partition size + let mut tmp = previous.ring_assignation_data.clone(); + tmp.sort(); + let partition_size = tmp + .into_iter() + .dedup_with_count() + .map(|(npart, node)| { + roles + .get(&node_id_vec[node as usize]) + .and_then(|p| p.0.as_ref().and_then(|r| r.capacity)) + .unwrap_or(0) / npart as u64 + }) + .min() + .unwrap_or(0); + + // By default, zone_redundancy is maximum possible value + let parameters = LayoutParameters { + zone_redundancy: ZoneRedundancy::Maximum, + }; + + Self { + version: previous.version, + replication_factor: previous.replication_factor, + partition_size, + parameters, + roles, + node_id_vec, + ring_assignment_data: previous.ring_assignation_data, + staging_parameters: Lww::new(parameters), + staging_roles, + staging_hash: [0u8; 32].into(), // will be set in the next migration + } + } + } + + fn multiply_all_capacities( + old_roles: LwwMap, + mul: u64, + ) -> LwwMap { + let mut new_roles = LwwMap::new(); + for (node, ts, role) in old_roles.items() { + let mut role = role.clone(); + if let NodeRoleV(Some(NodeRole { + capacity: Some(ref mut cap), + .. + })) = role + { + *cap *= mul; + } + new_roles.merge_raw(node, *ts, &role); + } + new_roles + } +} + +mod v010 { + use super::v09; + use crate::layout::CompactNodeType; + use garage_util::crdt::{Lww, LwwMap}; + use garage_util::data::Uuid; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeMap; + pub use v09::{LayoutParameters, NodeRole, NodeRoleV, ZoneRedundancy}; + + /// Number of old (non-live) versions to keep, see LayoutHistory::old_versions + pub const OLD_VERSION_COUNT: usize = 5; + + /// The history of cluster layouts, with trackers to keep a record + /// of which nodes are up-to-date to current cluster data + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutHistory { + /// The versions currently in use in the cluster + pub versions: Vec, + /// At most 5 of the previous versions, not used by the garage_table + /// module, but useful for the garage_block module to find data blocks + /// that have not yet been moved + pub old_versions: Vec, + + /// Update trackers + pub update_trackers: UpdateTrackers, + + /// Staged changes for the next version + pub staging: Lww, + } + + /// A version of the layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutVersion { + /// The number of this version + pub version: u64, + + /// Roles assigned to nodes in this version + pub roles: LwwMap, + /// Parameters used to compute the assignment currently given by + /// ring_assignment_data + pub parameters: LayoutParameters, + + /// The number of replicas for each data partition + pub replication_factor: usize, + /// This attribute is only used to retain the previously computed partition size, + /// to know to what extent does it change with the layout update. + pub partition_size: u64, + + /// node_id_vec: a vector of node IDs with a role assigned + /// in the system (this includes gateway nodes). + /// The order here is different than the vec stored by `roles`, because: + /// 1. non-gateway nodes are first so that they have lower numbers + /// 2. nodes that don't have a role are excluded (but they need to + /// stay in the CRDT as tombstones) + pub node_id_vec: Vec, + /// number of non-gateway nodes, which are the first ids in node_id_vec + pub nongateway_node_count: usize, + /// The assignation of data partitions to nodes, the values + /// are indices in node_id_vec + #[serde(with = "serde_bytes")] + pub ring_assignment_data: Vec, + } + + /// The staged changes for the next layout version + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutStaging { + /// Parameters to be used in the next partition assignment computation. + pub parameters: Lww, + /// Role changes which are staged for the next version of the layout + pub roles: LwwMap, + } + + /// The tracker of acknowlegments and data syncs around the cluster + #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq)] + pub struct UpdateTrackers { + /// The highest layout version number each node has ack'ed + pub ack_map: UpdateTracker, + /// The highest layout version number each node has synced data for + pub sync_map: UpdateTracker, + /// The highest layout version number each node has + /// ack'ed that all other nodes have synced data for + pub sync_ack_map: UpdateTracker, + } + + /// Generic update tracker struct + #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq)] + pub struct UpdateTracker(pub BTreeMap); + + impl garage_util::migrate::Migrate for LayoutHistory { + const VERSION_MARKER: &'static [u8] = b"G010lh"; + + type Previous = v09::ClusterLayout; + + fn migrate(previous: Self::Previous) -> Self { + let nongateway_node_count = previous + .node_id_vec + .iter() + .enumerate() + .filter(|(_, uuid)| { + let role = previous.roles.get(uuid); + matches!(role, Some(NodeRoleV(Some(role))) if role.capacity.is_some()) + }) + .map(|(i, _)| i + 1) + .max() + .unwrap_or(0); + + let version = LayoutVersion { + version: previous.version, + replication_factor: previous.replication_factor, + partition_size: previous.partition_size, + parameters: previous.parameters, + roles: previous.roles, + node_id_vec: previous.node_id_vec, + nongateway_node_count, + ring_assignment_data: previous.ring_assignment_data, + }; + let update_tracker = UpdateTracker( + version + .nongateway_nodes() + .iter() + .copied() + .map(|x| (x, version.version)) + .collect::>(), + ); + let staging = LayoutStaging { + parameters: previous.staging_parameters, + roles: previous.staging_roles, + }; + Self { + versions: vec![version], + old_versions: vec![], + update_trackers: UpdateTrackers { + ack_map: update_tracker.clone(), + sync_map: update_tracker.clone(), + sync_ack_map: update_tracker, + }, + staging: Lww::raw(previous.version, staging), + } + } + } +} + +pub use v010::*; + +// ---- utility functions ---- + +impl AutoCrdt for LayoutParameters { + const WARN_IF_DIFFERENT: bool = true; +} + +impl AutoCrdt for NodeRoleV { + const WARN_IF_DIFFERENT: bool = true; +} + +impl Crdt for LayoutStaging { + fn merge(&mut self, other: &LayoutStaging) { + self.parameters.merge(&other.parameters); + self.roles.merge(&other.roles); + } +} + +impl NodeRole { + pub fn capacity_string(&self) -> String { + match self.capacity { + Some(c) => ByteSize::b(c).to_string_as(false), + None => "gateway".to_string(), + } + } + + pub fn tags_string(&self) -> String { + self.tags.join(",") + } +} + +impl fmt::Display for ZoneRedundancy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ZoneRedundancy::Maximum => write!(f, "maximum"), + ZoneRedundancy::AtLeast(x) => write!(f, "{}", x), + } + } +} + +impl core::str::FromStr for ZoneRedundancy { + type Err = &'static str; + fn from_str(s: &str) -> Result { + match s { + "none" | "max" | "maximum" => Ok(ZoneRedundancy::Maximum), + x => { + let v = x + .parse::() + .map_err(|_| "zone redundancy must be 'none'/'max' or an integer")?; + Ok(ZoneRedundancy::AtLeast(v)) + } + } + } +} + +impl UpdateTracker { + fn merge(&mut self, other: &UpdateTracker) -> bool { + let mut changed = false; + for (k, v) in other.0.iter() { + if let Some(v_mut) = self.0.get_mut(k) { + if *v > *v_mut { + *v_mut = *v; + changed = true; + } + } else { + self.0.insert(*k, *v); + changed = true; + } + } + changed + } + + /// This bumps the update tracker for a given node up to the specified value. + /// This has potential impacts on the correctness of Garage and should only + /// be used in very specific circumstances. + pub fn set_max(&mut self, peer: Uuid, value: u64) -> bool { + match self.0.get_mut(&peer) { + Some(e) if *e < value => { + *e = value; + true + } + None => { + self.0.insert(peer, value); + true + } + _ => false, + } + } + + pub fn min_among(&self, storage_nodes: &[Uuid], min_version: u64) -> u64 { + storage_nodes + .iter() + .map(|x| self.get(x, min_version)) + .min() + .unwrap_or(min_version) + } + + pub fn get(&self, node: &Uuid, min_version: u64) -> u64 { + self.0.get(node).copied().unwrap_or(min_version) + } +} + +impl UpdateTrackers { + pub(crate) fn merge(&mut self, other: &UpdateTrackers) -> bool { + let c1 = self.ack_map.merge(&other.ack_map); + let c2 = self.sync_map.merge(&other.sync_map); + let c3 = self.sync_ack_map.merge(&other.sync_ack_map); + c1 || c2 || c3 + } +} diff --git a/src/rpc/layout/test.rs b/src/rpc/layout/test.rs new file mode 100644 index 00000000..5462160b --- /dev/null +++ b/src/rpc/layout/test.rs @@ -0,0 +1,158 @@ +use std::cmp::min; +use std::collections::HashMap; + +use garage_util::crdt::Crdt; +use garage_util::error::*; + +use crate::layout::*; +use crate::replication_mode::ReplicationFactor; + +// This function checks that the partition size S computed is at least better than the +// one given by a very naive algorithm. To do so, we try to run the naive algorithm +// assuming a partition size of S+1. If we succeed, it means that the optimal assignment +// was not optimal. The naive algorithm is the following : +// - we compute the max number of partitions associated to every node, capped at the +// partition number. It gives the number of tokens of every node. +// - every zone has a number of tokens equal to the sum of the tokens of its nodes. +// - we cycle over the partitions and associate zone tokens while respecting the +// zone redundancy constraint. +// NOTE: the naive algorithm is not optimal. Counter example: +// take nb_partition = 3 ; replication_factor = 5; redundancy = 4; +// number of tokens by zone : (A, 4), (B,1), (C,4), (D, 4), (E, 2) +// With these parameters, the naive algo fails, whereas there is a solution: +// (A,A,C,D,E) , (A,B,C,D,D) (A,C,C,D,E) +fn check_against_naive(cl: &LayoutVersion) -> Result { + let over_size = cl.partition_size + 1; + let mut zone_token = HashMap::::new(); + + let (zones, zone_to_id) = cl.generate_nongateway_zone_ids()?; + + if zones.is_empty() { + return Ok(false); + } + + for z in zones.iter() { + zone_token.insert(z.clone(), 0); + } + for uuid in cl.nongateway_nodes() { + let z = cl.expect_get_node_zone(&uuid); + let c = cl.expect_get_node_capacity(&uuid); + zone_token.insert( + z.to_string(), + zone_token[z] + min(NB_PARTITIONS, (c / over_size) as usize), + ); + } + + // For every partition, we count the number of zone already associated and + // the name of the last zone associated + + let mut id_zone_token = vec![0; zones.len()]; + for (z, t) in zone_token.iter() { + id_zone_token[zone_to_id[z]] = *t; + } + + let mut nb_token = vec![0; NB_PARTITIONS]; + let mut last_zone = vec![zones.len(); NB_PARTITIONS]; + + let mut curr_zone = 0; + + let redundancy = cl.effective_zone_redundancy(); + + for replic in 0..cl.replication_factor { + for p in 0..NB_PARTITIONS { + while id_zone_token[curr_zone] == 0 + || (last_zone[p] == curr_zone + && redundancy - nb_token[p] <= cl.replication_factor - replic) + { + curr_zone += 1; + if curr_zone >= zones.len() { + return Ok(true); + } + } + id_zone_token[curr_zone] -= 1; + if last_zone[p] != curr_zone { + nb_token[p] += 1; + last_zone[p] = curr_zone; + } + } + } + + return Ok(false); +} + +fn show_msg(msg: &Message) { + for s in msg.iter() { + println!("{}", s); + } +} + +fn update_layout( + cl: &mut LayoutHistory, + node_capacity_vec: &[u64], + node_zone_vec: &[&'static str], + zone_redundancy: usize, +) { + let staging = cl.staging.get_mut(); + + for (i, (capacity, zone)) in node_capacity_vec + .iter() + .zip(node_zone_vec.iter()) + .enumerate() + { + let node_id = [i as u8; 32].into(); + + let update = staging.roles.update_mutator( + node_id, + NodeRoleV(Some(NodeRole { + zone: zone.to_string(), + capacity: Some(*capacity), + tags: (vec![]), + })), + ); + staging.roles.merge(&update); + } + staging.parameters.update(LayoutParameters { + zone_redundancy: ZoneRedundancy::AtLeast(zone_redundancy), + }); +} + +#[test] +fn test_assignment() { + let mut node_capacity_vec = vec![4000, 1000, 2000]; + let mut node_zone_vec = vec!["A", "B", "C"]; + + let mut cl = LayoutHistory::new(ReplicationFactor::new(3).unwrap()); + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 3); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![4000, 1000, 1000, 3000, 1000, 1000, 2000, 10000, 2000]; + node_zone_vec = vec!["A", "B", "C", "C", "C", "B", "G", "H", "I"]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 2); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![4000, 1000, 2000, 7000, 1000, 1000, 2000, 10000, 2000]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 3); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![ + 4000000, 4000000, 2000000, 7000000, 1000000, 9000000, 2000000, 10000, 2000000, + ]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 1); + let v = cl.current().version; + let (cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); +} diff --git a/src/rpc/layout.rs b/src/rpc/layout/version.rs similarity index 55% rename from src/rpc/layout.rs rename to src/rpc/layout/version.rs index e02a180b..a02fce89 100644 --- a/src/rpc/layout.rs +++ b/src/rpc/layout/version.rs @@ -1,375 +1,55 @@ -use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; -use std::fmt; +use std::convert::TryInto; use bytesize::ByteSize; use itertools::Itertools; -use garage_util::crdt::{AutoCrdt, Crdt, Lww, LwwMap}; +use garage_util::crdt::{Crdt, LwwMap}; use garage_util::data::*; -use garage_util::encode::nonversioned_encode; use garage_util::error::*; -use crate::graph_algo::*; - -use crate::ring::*; - -use std::convert::TryInto; - -const NB_PARTITIONS: usize = 1usize << PARTITION_BITS; +use super::graph_algo::*; +use super::*; // The Message type will be used to collect information on the algorithm. -type Message = Vec; +pub type Message = Vec; -mod v08 { - use crate::ring::CompactNodeType; - use garage_util::crdt::LwwMap; - use garage_util::data::{Hash, Uuid}; - use serde::{Deserialize, Serialize}; - - /// The layout of the cluster, i.e. the list of roles - /// which are assigned to each cluster node - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct ClusterLayout { - pub version: u64, - - pub replication_factor: usize, - pub roles: LwwMap, - - /// node_id_vec: a vector of node IDs with a role assigned - /// in the system (this includes gateway nodes). - /// The order here is different than the vec stored by `roles`, because: - /// 1. non-gateway nodes are first so that they have lower numbers - /// 2. nodes that don't have a role are excluded (but they need to - /// stay in the CRDT as tombstones) - pub node_id_vec: Vec, - /// the assignation of data partitions to node, the values - /// are indices in node_id_vec - #[serde(with = "serde_bytes")] - pub ring_assignation_data: Vec, - - /// Role changes which are staged for the next version of the layout - pub staging: LwwMap, - pub staging_hash: Hash, - } - - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] - pub struct NodeRoleV(pub Option); - - /// The user-assigned roles of cluster nodes - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] - pub struct NodeRole { - /// Datacenter at which this entry belong. This information is used to - /// perform a better geodistribution - pub zone: String, - /// The capacity of the node - /// If this is set to None, the node does not participate in storing data for the system - /// and is only active as an API gateway to other nodes - pub capacity: Option, - /// A set of tags to recognize the node - pub tags: Vec, - } - - impl garage_util::migrate::InitialFormat for ClusterLayout {} -} - -mod v09 { - use super::v08; - use crate::ring::CompactNodeType; - use garage_util::crdt::{Lww, LwwMap}; - use garage_util::data::{Hash, Uuid}; - use serde::{Deserialize, Serialize}; - pub use v08::{NodeRole, NodeRoleV}; - - /// The layout of the cluster, i.e. the list of roles - /// which are assigned to each cluster node - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct ClusterLayout { - pub version: u64, - - pub replication_factor: usize, - - /// This attribute is only used to retain the previously computed partition size, - /// to know to what extent does it change with the layout update. - pub partition_size: u64, - /// Parameters used to compute the assignment currently given by - /// ring_assignment_data - pub parameters: LayoutParameters, - - pub roles: LwwMap, - - /// see comment in v08::ClusterLayout - pub node_id_vec: Vec, - /// see comment in v08::ClusterLayout - #[serde(with = "serde_bytes")] - pub ring_assignment_data: Vec, - - /// Parameters to be used in the next partition assignment computation. - pub staging_parameters: Lww, - /// Role changes which are staged for the next version of the layout - pub staging_roles: LwwMap, - pub staging_hash: Hash, - } - - /// This struct is used to set the parameters to be used in the assignment computation - /// algorithm. It is stored as a Crdt. - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] - pub struct LayoutParameters { - pub zone_redundancy: ZoneRedundancy, - } - - /// Zone redundancy: if set to AtLeast(x), the layout calculation will aim to store copies - /// of each partition on at least that number of different zones. - /// Otherwise, copies will be stored on the maximum possible number of zones. - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] - pub enum ZoneRedundancy { - AtLeast(usize), - Maximum, - } - - impl garage_util::migrate::Migrate for ClusterLayout { - const VERSION_MARKER: &'static [u8] = b"G09layout"; - - type Previous = v08::ClusterLayout; - - fn migrate(previous: Self::Previous) -> Self { - use itertools::Itertools; - - // In the old layout, capacities are in an arbitrary unit, - // but in the new layout they are in bytes. - // Here we arbitrarily multiply everything by 1G, - // such that 1 old capacity unit = 1GB in the new units. - // This is totally arbitrary and won't work for most users. - let cap_mul = 1024 * 1024 * 1024; - let roles = multiply_all_capacities(previous.roles, cap_mul); - let staging_roles = multiply_all_capacities(previous.staging, cap_mul); - let node_id_vec = previous.node_id_vec; - - // Determine partition size - let mut tmp = previous.ring_assignation_data.clone(); - tmp.sort(); - let partition_size = tmp - .into_iter() - .dedup_with_count() - .map(|(npart, node)| { - roles - .get(&node_id_vec[node as usize]) - .and_then(|p| p.0.as_ref().and_then(|r| r.capacity)) - .unwrap_or(0) / npart as u64 - }) - .min() - .unwrap_or(0); - - // By default, zone_redundancy is maximum possible value - let parameters = LayoutParameters { - zone_redundancy: ZoneRedundancy::Maximum, - }; - - let mut res = Self { - version: previous.version, - replication_factor: previous.replication_factor, - partition_size, - parameters, - roles, - node_id_vec, - ring_assignment_data: previous.ring_assignation_data, - staging_parameters: Lww::new(parameters), - staging_roles, - staging_hash: [0u8; 32].into(), - }; - res.staging_hash = res.calculate_staging_hash(); - res - } - } - - fn multiply_all_capacities( - old_roles: LwwMap, - mul: u64, - ) -> LwwMap { - let mut new_roles = LwwMap::new(); - for (node, ts, role) in old_roles.items() { - let mut role = role.clone(); - if let NodeRoleV(Some(NodeRole { - capacity: Some(ref mut cap), - .. - })) = role - { - *cap *= mul; - } - new_roles.merge_raw(node, *ts, &role); - } - new_roles - } -} - -pub use v09::*; - -impl AutoCrdt for LayoutParameters { - const WARN_IF_DIFFERENT: bool = true; -} - -impl AutoCrdt for NodeRoleV { - const WARN_IF_DIFFERENT: bool = true; -} - -impl NodeRole { - pub fn capacity_string(&self) -> String { - match self.capacity { - Some(c) => ByteSize::b(c).to_string_as(false), - None => "gateway".to_string(), - } - } - - pub fn tags_string(&self) -> String { - self.tags.join(",") - } -} - -impl fmt::Display for ZoneRedundancy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ZoneRedundancy::Maximum => write!(f, "maximum"), - ZoneRedundancy::AtLeast(x) => write!(f, "{}", x), - } - } -} - -impl core::str::FromStr for ZoneRedundancy { - type Err = &'static str; - fn from_str(s: &str) -> Result { - match s { - "none" | "max" | "maximum" => Ok(ZoneRedundancy::Maximum), - x => { - let v = x - .parse::() - .map_err(|_| "zone redundancy must be 'none'/'max' or an integer")?; - Ok(ZoneRedundancy::AtLeast(v)) - } - } - } -} - -// Implementation of the ClusterLayout methods unrelated to the assignment algorithm. -impl ClusterLayout { +impl LayoutVersion { pub fn new(replication_factor: usize) -> Self { // We set the default zone redundancy to be Maximum, meaning that the maximum // possible value will be used depending on the cluster topology let parameters = LayoutParameters { zone_redundancy: ZoneRedundancy::Maximum, }; - let staging_parameters = Lww::::new(parameters); - let empty_lwwmap = LwwMap::new(); - - let mut ret = ClusterLayout { + LayoutVersion { version: 0, replication_factor, partition_size: 0, roles: LwwMap::new(), node_id_vec: Vec::new(), + nongateway_node_count: 0, ring_assignment_data: Vec::new(), parameters, - staging_parameters, - staging_roles: empty_lwwmap, - staging_hash: [0u8; 32].into(), - }; - ret.staging_hash = ret.calculate_staging_hash(); - ret - } - - fn calculate_staging_hash(&self) -> Hash { - let hashed_tuple = (&self.staging_roles, &self.staging_parameters); - blake2sum(&nonversioned_encode(&hashed_tuple).unwrap()[..]) - } - - pub fn merge(&mut self, other: &ClusterLayout) -> bool { - match other.version.cmp(&self.version) { - Ordering::Greater => { - *self = other.clone(); - true - } - Ordering::Equal => { - self.staging_parameters.merge(&other.staging_parameters); - self.staging_roles.merge(&other.staging_roles); - - let new_staging_hash = self.calculate_staging_hash(); - let changed = new_staging_hash != self.staging_hash; - - self.staging_hash = new_staging_hash; - - changed - } - Ordering::Less => false, } } - pub fn apply_staged_changes(mut self, version: Option) -> Result<(Self, Message), Error> { - match version { - None => { - let error = r#" -Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. -To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. - "#; - return Err(Error::Message(error.into())); - } - Some(v) => { - if v != self.version + 1 { - return Err(Error::Message("Invalid new layout version".into())); - } - } - } + // ===================== accessors ====================== - self.roles.merge(&self.staging_roles); - self.roles.retain(|(_, _, v)| v.0.is_some()); - self.parameters = *self.staging_parameters.get(); - - self.staging_roles.clear(); - self.staging_hash = self.calculate_staging_hash(); - - let msg = self.calculate_partition_assignment()?; - - self.version += 1; - - Ok((self, msg)) - } - - pub fn revert_staged_changes(mut self, version: Option) -> Result { - match version { - None => { - let error = r#" -Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. -To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. - "#; - return Err(Error::Message(error.into())); - } - Some(v) => { - if v != self.version + 1 { - return Err(Error::Message("Invalid new layout version".into())); - } - } - } - - self.staging_roles.clear(); - self.staging_parameters.update(self.parameters); - self.staging_hash = self.calculate_staging_hash(); - - self.version += 1; - - Ok(self) - } - - /// Returns a list of IDs of nodes that currently have - /// a role in the cluster - pub fn node_ids(&self) -> &[Uuid] { + /// Returns a list of IDs of nodes that have a role in this + /// version of the cluster layout, including gateway nodes + pub fn all_nodes(&self) -> &[Uuid] { &self.node_id_vec[..] } - pub fn num_nodes(&self) -> usize { - self.node_id_vec.len() + /// Returns a list of IDs of nodes that have a storage capacity + /// assigned in this version of the cluster layout + pub fn nongateway_nodes(&self) -> &[Uuid] { + &self.node_id_vec[..self.nongateway_node_count] } - /// Returns the role of a node in the layout + /// Returns the role of a node in the layout, if it has one pub fn node_role(&self, node: &Uuid) -> Option<&NodeRole> { match self.roles.get(node) { Some(NodeRoleV(Some(v))) => Some(v), @@ -377,41 +57,23 @@ To know the correct value of the new layout version, invoke `garage layout show` } } - /// Returns the uuids of the non_gateway nodes in self.node_id_vec. - fn nongateway_nodes(&self) -> Vec { - let mut result = Vec::::new(); - for uuid in self.node_id_vec.iter() { - match self.node_role(uuid) { - Some(role) if role.capacity.is_some() => result.push(*uuid), - _ => (), - } - } - result - } - - /// Given a node uuids, this function returns the label of its zone - fn get_node_zone(&self, uuid: &Uuid) -> Result { - match self.node_role(uuid) { - Some(role) => Ok(role.zone.clone()), - _ => Err(Error::Message( - "The Uuid does not correspond to a node present in the cluster.".into(), - )), - } - } - - /// Given a node uuids, this function returns its capacity or fails if it does not have any - pub fn get_node_capacity(&self, uuid: &Uuid) -> Result { + /// Returns the capacity of a node in the layout, if it has one + pub fn get_node_capacity(&self, uuid: &Uuid) -> Option { match self.node_role(uuid) { Some(NodeRole { capacity: Some(cap), zone: _, tags: _, - }) => Ok(*cap), - _ => Err(Error::Message( - "The Uuid does not correspond to a node present in the \ - cluster or this node does not have a positive capacity." - .into(), - )), + }) => Some(*cap), + _ => None, + } + } + + /// Given a node uuids, this function returns the label of its zone if it has one + pub fn get_node_zone(&self, uuid: &Uuid) -> Option<&str> { + match self.node_role(uuid) { + Some(role) => Some(&role.zone), + _ => None, } } @@ -435,17 +97,65 @@ To know the correct value of the new layout version, invoke `garage layout show` )) } + /// Get the partition in which data would fall on + pub fn partition_of(&self, position: &Hash) -> Partition { + let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); + top >> (16 - PARTITION_BITS) + } + + /// Get the list of partitions and the first hash of a partition key that would fall in it + pub fn partitions(&self) -> impl Iterator + '_ { + (0..(1 << PARTITION_BITS)).map(|i| { + let top = (i as u16) << (16 - PARTITION_BITS); + let mut location = [0u8; 32]; + location[..2].copy_from_slice(&u16::to_be_bytes(top)[..]); + (i as u16, Hash::from(location)) + }) + } + + /// Return the n servers in which data for this hash should be replicated + pub fn nodes_of(&self, position: &Hash, n: usize) -> impl Iterator + '_ { + assert_eq!(n, self.replication_factor); + + let data = &self.ring_assignment_data; + + let partition_nodes = if data.len() == self.replication_factor * (1 << PARTITION_BITS) { + let partition_idx = self.partition_of(position) as usize; + let partition_start = partition_idx * self.replication_factor; + let partition_end = (partition_idx + 1) * self.replication_factor; + &data[partition_start..partition_end] + } else { + warn!("Ring not yet ready, read/writes will be lost!"); + &[] + }; + + partition_nodes + .iter() + .map(move |i| self.node_id_vec[*i as usize]) + } + + // ===================== internal information extractors ====================== + + pub(crate) fn expect_get_node_capacity(&self, uuid: &Uuid) -> u64 { + self.get_node_capacity(uuid) + .expect("non-gateway node with zero capacity") + } + + pub(crate) fn expect_get_node_zone(&self, uuid: &Uuid) -> &str { + self.get_node_zone(uuid).expect("node without a zone") + } + /// Returns the sum of capacities of non gateway nodes in the cluster - fn get_total_capacity(&self) -> Result { + fn get_total_capacity(&self) -> u64 { let mut total_capacity = 0; - for uuid in self.nongateway_nodes().iter() { - total_capacity += self.get_node_capacity(uuid)?; + for uuid in self.nongateway_nodes() { + total_capacity += self.expect_get_node_capacity(uuid); } - Ok(total_capacity) + total_capacity } /// Returns the effective value of the zone_redundancy parameter - fn effective_zone_redundancy(&self) -> usize { + pub(crate) fn effective_zone_redundancy(&self) -> usize { match self.parameters.zone_redundancy { ZoneRedundancy::AtLeast(v) => v, ZoneRedundancy::Maximum => { @@ -465,10 +175,14 @@ To know the correct value of the new layout version, invoke `garage layout show` /// (assignment, roles, parameters, partition size) /// returns true if consistent, false if error pub fn check(&self) -> Result<(), String> { - // Check that the hash of the staging data is correct - let staging_hash = self.calculate_staging_hash(); - if staging_hash != self.staging_hash { - return Err("staging_hash is incorrect".into()); + // Check that the assignment data has the correct length + let expected_assignment_data_len = (1 << PARTITION_BITS) * self.replication_factor; + if self.ring_assignment_data.len() != expected_assignment_data_len { + return Err(format!( + "ring_assignment_data has incorrect length {} instead of {}", + self.ring_assignment_data.len(), + expected_assignment_data_len + )); } // Check that node_id_vec contains the correct list of nodes @@ -486,16 +200,6 @@ To know the correct value of the new layout version, invoke `garage layout show` return Err(format!("node_id_vec does not contain the correct set of nodes\nnode_id_vec: {:?}\nexpected: {:?}", node_id_vec, expected_nodes)); } - // Check that the assignment data has the correct length - let expected_assignment_data_len = (1 << PARTITION_BITS) * self.replication_factor; - if self.ring_assignment_data.len() != expected_assignment_data_len { - return Err(format!( - "ring_assignment_data has incorrect length {} instead of {}", - self.ring_assignment_data.len(), - expected_assignment_data_len - )); - } - // Check that the assigned nodes are correct identifiers // of nodes that are assigned a role // and that role is not the role of a gateway nodes @@ -524,10 +228,7 @@ To know the correct value of the new layout version, invoke `garage layout show` // Check that every partition is spread over at least zone_redundancy zones. let zones_of_p = nodes_of_p .iter() - .map(|n| { - self.get_node_zone(&self.node_id_vec[*n as usize]) - .expect("Zone not found.") - }) + .map(|n| self.expect_get_node_zone(&self.node_id_vec[*n as usize])) .collect::>(); if zones_of_p.iter().unique().count() < zone_redundancy { return Err(format!( @@ -546,7 +247,7 @@ To know the correct value of the new layout version, invoke `garage layout show` if *usage > 0 { let uuid = self.node_id_vec[n]; let partusage = usage * self.partition_size; - let nodecap = self.get_node_capacity(&uuid).unwrap(); + let nodecap = self.expect_get_node_capacity(&uuid); if partusage > nodecap { return Err(format!( "node usage ({}) is bigger than node capacity ({})", @@ -574,12 +275,24 @@ To know the correct value of the new layout version, invoke `garage layout show` Ok(()) } -} -// ==================================================================================== + // ================== updates to layout, internals =================== + + pub(crate) fn calculate_next_version( + mut self, + staging: &LayoutStaging, + ) -> Result<(Self, Message), Error> { + self.version += 1; + + self.roles.merge(&staging.roles); + self.roles.retain(|(_, _, v)| v.0.is_some()); + self.parameters = *staging.parameters.get(); + + let msg = self.calculate_partition_assignment()?; + + Ok((self, msg)) + } -// Implementation of the ClusterLayout methods related to the assignment algorithm. -impl ClusterLayout { /// This function calculates a new partition-to-node assignment. /// The computed assignment respects the node replication factor /// and the zone redundancy parameter It maximizes the capacity of a @@ -609,12 +322,12 @@ impl ClusterLayout { // to use them as indices in the flow graphs. let (id_to_zone, zone_to_id) = self.generate_nongateway_zone_ids()?; - let nb_nongateway_nodes = self.nongateway_nodes().len(); - if nb_nongateway_nodes < self.replication_factor { + if self.nongateway_nodes().len() < self.replication_factor { return Err(Error::Message(format!( "The number of nodes with positive \ capacity ({}) is smaller than the replication factor ({}).", - nb_nongateway_nodes, self.replication_factor + self.nongateway_nodes().len(), + self.replication_factor ))); } if id_to_zone.len() < zone_redundancy { @@ -712,12 +425,14 @@ impl ClusterLayout { .map(|(k, _, _)| *k) .collect(); - let mut new_node_id_vec = Vec::::new(); - new_node_id_vec.extend(new_non_gateway_nodes); - new_node_id_vec.extend(new_gateway_nodes); + let old_node_id_vec = std::mem::take(&mut self.node_id_vec); - let old_node_id_vec = self.node_id_vec.clone(); - self.node_id_vec = new_node_id_vec.clone(); + self.nongateway_node_count = new_non_gateway_nodes.len(); + self.node_id_vec.clear(); + self.node_id_vec.extend(new_non_gateway_nodes); + self.node_id_vec.extend(new_gateway_nodes); + + let new_node_id_vec = &self.node_id_vec; // (2) We retrieve the old association // We rewrite the old association with the new indices. We only consider partition @@ -756,7 +471,7 @@ impl ClusterLayout { } } - // We write the ring + // We clear the ring assignment data self.ring_assignment_data = Vec::::new(); Ok(Some(old_assignment)) @@ -764,7 +479,9 @@ impl ClusterLayout { /// This function generates ids for the zone of the nodes appearing in /// self.node_id_vec. - fn generate_nongateway_zone_ids(&self) -> Result<(Vec, HashMap), Error> { + pub(crate) fn generate_nongateway_zone_ids( + &self, + ) -> Result<(Vec, HashMap), Error> { let mut id_to_zone = Vec::::new(); let mut zone_to_id = HashMap::::new(); @@ -790,14 +507,14 @@ impl ClusterLayout { g.compute_maximal_flow()?; if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 { return Err(Error::Message( - "The storage capacity of he cluster is to small. It is \ + "The storage capacity of the cluster is too small. It is \ impossible to store partitions of size 1." .into(), )); } let mut s_down = 1; - let mut s_up = self.get_total_capacity()?; + let mut s_up = self.get_total_capacity(); while s_down + 1 < s_up { g = self.generate_flow_graph( (s_down + s_up) / 2, @@ -846,7 +563,7 @@ impl ClusterLayout { zone_redundancy: usize, ) -> Result, Error> { let vertices = - ClusterLayout::generate_graph_vertices(zone_to_id.len(), self.nongateway_nodes().len()); + LayoutVersion::generate_graph_vertices(zone_to_id.len(), self.nongateway_nodes().len()); let mut g = Graph::::new(&vertices); let nb_zones = zone_to_id.len(); for p in 0..NB_PARTITIONS { @@ -866,8 +583,8 @@ impl ClusterLayout { } } for n in 0..self.nongateway_nodes().len() { - let node_capacity = self.get_node_capacity(&self.node_id_vec[n])?; - let node_zone = zone_to_id[&self.get_node_zone(&self.node_id_vec[n])?]; + let node_capacity = self.expect_get_node_capacity(&self.node_id_vec[n]); + let node_zone = zone_to_id[self.expect_get_node_zone(&self.node_id_vec[n])]; g.add_edge(Vertex::N(n), Vertex::Sink, node_capacity / partition_size)?; for p in 0..NB_PARTITIONS { if !exclude_assoc.contains(&(p, n)) { @@ -913,7 +630,7 @@ impl ClusterLayout { // The algorithm is such that it will start with the flow that we just computed // and find ameliorating paths from that. for (p, n) in exclude_edge.iter() { - let node_zone = zone_to_id[&self.get_node_zone(&self.node_id_vec[*n])?]; + let node_zone = zone_to_id[self.expect_get_node_zone(&self.node_id_vec[*n])]; g.add_edge(Vertex::PZ(*p, node_zone), Vertex::N(*n), 1)?; } g.compute_maximal_flow()?; @@ -933,8 +650,11 @@ impl ClusterLayout { let mut cost = CostFunction::new(); for (p, assoc_p) in prev_assign.iter().enumerate() { for n in assoc_p.iter() { - let node_zone = zone_to_id[&self.get_node_zone(&self.node_id_vec[*n])?]; - cost.insert((Vertex::PZ(p, node_zone), Vertex::N(*n)), -1); + if let Some(&node_zone) = + zone_to_id.get(self.expect_get_node_zone(&self.node_id_vec[*n])) + { + cost.insert((Vertex::PZ(p, node_zone), Vertex::N(*n)), -1); + } } } @@ -988,7 +708,7 @@ impl ClusterLayout { let mut msg = Message::new(); let used_cap = self.partition_size * NB_PARTITIONS as u64 * self.replication_factor as u64; - let total_cap = self.get_total_capacity()?; + let total_cap = self.get_total_capacity(); let percent_cap = 100.0 * (used_cap as f32) / (total_cap as f32); msg.push(format!( "Usable capacity / total cluster capacity: {} / {} ({:.1} %)", @@ -1034,8 +754,11 @@ impl ClusterLayout { if let Some(prev_assign) = prev_assign_opt { let mut old_zones_of_p = Vec::::new(); for n in prev_assign[p].iter() { - old_zones_of_p - .push(zone_to_id[&self.get_node_zone(&self.node_id_vec[*n])?]); + if let Some(&zone_id) = + zone_to_id.get(self.expect_get_node_zone(&self.node_id_vec[*n])) + { + old_zones_of_p.push(zone_id); + } } if !old_zones_of_p.contains(&z) { new_partitions_zone[z] += 1; @@ -1077,7 +800,7 @@ impl ClusterLayout { for z in 0..id_to_zone.len() { let mut nodes_of_z = Vec::::new(); for n in 0..storing_nodes.len() { - if self.get_node_zone(&self.node_id_vec[n])? == id_to_zone[z] { + if self.expect_get_node_zone(&self.node_id_vec[n]) == id_to_zone[z] { nodes_of_z.push(n); } } @@ -1091,13 +814,13 @@ impl ClusterLayout { let available_cap_z: u64 = self.partition_size * replicated_partitions as u64; let mut total_cap_z = 0; for n in nodes_of_z.iter() { - total_cap_z += self.get_node_capacity(&self.node_id_vec[*n])?; + total_cap_z += self.expect_get_node_capacity(&self.node_id_vec[*n]); } let percent_cap_z = 100.0 * (available_cap_z as f32) / (total_cap_z as f32); for n in nodes_of_z.iter() { let available_cap_n = stored_partitions[*n] as u64 * self.partition_size; - let total_cap_n = self.get_node_capacity(&self.node_id_vec[*n])?; + let total_cap_n = self.expect_get_node_capacity(&self.node_id_vec[*n]); let tags_n = (self.node_role(&self.node_id_vec[*n]).ok_or(""))?.tags_string(); table.push(format!( " {:?}\t{}\t{} ({} new)\t{}\t{} ({:.1}%)", @@ -1127,167 +850,3 @@ impl ClusterLayout { Ok(msg) } } - -// ==================================================================================== - -#[cfg(test)] -mod tests { - use super::{Error, *}; - use std::cmp::min; - - // This function checks that the partition size S computed is at least better than the - // one given by a very naive algorithm. To do so, we try to run the naive algorithm - // assuming a partion size of S+1. If we succed, it means that the optimal assignment - // was not optimal. The naive algorithm is the following : - // - we compute the max number of partitions associated to every node, capped at the - // partition number. It gives the number of tokens of every node. - // - every zone has a number of tokens equal to the sum of the tokens of its nodes. - // - we cycle over the partitions and associate zone tokens while respecting the - // zone redundancy constraint. - // NOTE: the naive algorithm is not optimal. Counter example: - // take nb_partition = 3 ; replication_factor = 5; redundancy = 4; - // number of tokens by zone : (A, 4), (B,1), (C,4), (D, 4), (E, 2) - // With these parameters, the naive algo fails, whereas there is a solution: - // (A,A,C,D,E) , (A,B,C,D,D) (A,C,C,D,E) - fn check_against_naive(cl: &ClusterLayout) -> Result { - let over_size = cl.partition_size + 1; - let mut zone_token = HashMap::::new(); - - let (zones, zone_to_id) = cl.generate_nongateway_zone_ids()?; - - if zones.is_empty() { - return Ok(false); - } - - for z in zones.iter() { - zone_token.insert(z.clone(), 0); - } - for uuid in cl.nongateway_nodes().iter() { - let z = cl.get_node_zone(uuid)?; - let c = cl.get_node_capacity(uuid)?; - zone_token.insert( - z.clone(), - zone_token[&z] + min(NB_PARTITIONS, (c / over_size) as usize), - ); - } - - // For every partition, we count the number of zone already associated and - // the name of the last zone associated - - let mut id_zone_token = vec![0; zones.len()]; - for (z, t) in zone_token.iter() { - id_zone_token[zone_to_id[z]] = *t; - } - - let mut nb_token = vec![0; NB_PARTITIONS]; - let mut last_zone = vec![zones.len(); NB_PARTITIONS]; - - let mut curr_zone = 0; - - let redundancy = cl.effective_zone_redundancy(); - - for replic in 0..cl.replication_factor { - for p in 0..NB_PARTITIONS { - while id_zone_token[curr_zone] == 0 - || (last_zone[p] == curr_zone - && redundancy - nb_token[p] <= cl.replication_factor - replic) - { - curr_zone += 1; - if curr_zone >= zones.len() { - return Ok(true); - } - } - id_zone_token[curr_zone] -= 1; - if last_zone[p] != curr_zone { - nb_token[p] += 1; - last_zone[p] = curr_zone; - } - } - } - - return Ok(false); - } - - fn show_msg(msg: &Message) { - for s in msg.iter() { - println!("{}", s); - } - } - - fn update_layout( - cl: &mut ClusterLayout, - node_id_vec: &Vec, - node_capacity_vec: &Vec, - node_zone_vec: &Vec, - zone_redundancy: usize, - ) { - for i in 0..node_id_vec.len() { - if let Some(x) = FixedBytes32::try_from(&[i as u8; 32]) { - cl.node_id_vec.push(x); - } - - let update = cl.staging_roles.update_mutator( - cl.node_id_vec[i], - NodeRoleV(Some(NodeRole { - zone: (node_zone_vec[i].to_string()), - capacity: (Some(node_capacity_vec[i])), - tags: (vec![]), - })), - ); - cl.staging_roles.merge(&update); - } - cl.staging_parameters.update(LayoutParameters { - zone_redundancy: ZoneRedundancy::AtLeast(zone_redundancy), - }); - cl.staging_hash = cl.calculate_staging_hash(); - } - - #[test] - fn test_assignment() { - let mut node_id_vec = vec![1, 2, 3]; - let mut node_capacity_vec = vec![4000, 1000, 2000]; - let mut node_zone_vec = vec!["A", "B", "C"] - .into_iter() - .map(|x| x.to_string()) - .collect(); - - let mut cl = ClusterLayout::new(3); - update_layout(&mut cl, &node_id_vec, &node_capacity_vec, &node_zone_vec, 3); - let v = cl.version; - let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); - show_msg(&msg); - assert_eq!(cl.check(), Ok(())); - assert!(matches!(check_against_naive(&cl), Ok(true))); - - node_id_vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9]; - node_capacity_vec = vec![4000, 1000, 1000, 3000, 1000, 1000, 2000, 10000, 2000]; - node_zone_vec = vec!["A", "B", "C", "C", "C", "B", "G", "H", "I"] - .into_iter() - .map(|x| x.to_string()) - .collect(); - update_layout(&mut cl, &node_id_vec, &node_capacity_vec, &node_zone_vec, 2); - let v = cl.version; - let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); - show_msg(&msg); - assert_eq!(cl.check(), Ok(())); - assert!(matches!(check_against_naive(&cl), Ok(true))); - - node_capacity_vec = vec![4000, 1000, 2000, 7000, 1000, 1000, 2000, 10000, 2000]; - update_layout(&mut cl, &node_id_vec, &node_capacity_vec, &node_zone_vec, 3); - let v = cl.version; - let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); - show_msg(&msg); - assert_eq!(cl.check(), Ok(())); - assert!(matches!(check_against_naive(&cl), Ok(true))); - - node_capacity_vec = vec![ - 4000000, 4000000, 2000000, 7000000, 1000000, 9000000, 2000000, 10000, 2000000, - ]; - update_layout(&mut cl, &node_id_vec, &node_capacity_vec, &node_zone_vec, 1); - let v = cl.version; - let (cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); - show_msg(&msg); - assert_eq!(cl.check(), Ok(())); - assert!(matches!(check_against_naive(&cl), Ok(true))); - } -} diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index a5f8fc6e..b5b31c05 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -11,10 +11,8 @@ mod consul; #[cfg(feature = "kubernetes-discovery")] mod kubernetes; -pub mod graph_algo; pub mod layout; pub mod replication_mode; -pub mod ring; pub mod system; pub mod rpc_helper; diff --git a/src/rpc/replication_mode.rs b/src/rpc/replication_mode.rs index e244e063..a3a94085 100644 --- a/src/rpc/replication_mode.rs +++ b/src/rpc/replication_mode.rs @@ -1,57 +1,94 @@ -#[derive(Clone, Copy)] -pub enum ReplicationMode { - None, - TwoWay, - TwoWayDangerous, - ThreeWay, - ThreeWayDegraded, - ThreeWayDangerous, +use garage_util::config::Config; +use garage_util::crdt::AutoCrdt; +use garage_util::error::*; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ReplicationFactor(usize); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ConsistencyMode { + /// Read- and Write-quorum are 1 + Dangerous, + /// Read-quorum is 1 + Degraded, + /// Read- and Write-quorum are determined for read-after-write-consistency + #[default] + Consistent, } -impl ReplicationMode { - pub fn parse(v: &str) -> Option { - match v { - "none" | "1" => Some(Self::None), - "2" => Some(Self::TwoWay), - "2-dangerous" => Some(Self::TwoWayDangerous), - "3" => Some(Self::ThreeWay), - "3-degraded" => Some(Self::ThreeWayDegraded), - "3-dangerous" => Some(Self::ThreeWayDangerous), - _ => None, - } +impl ConsistencyMode { + pub fn parse(s: &str) -> Option { + serde_json::from_value(serde_json::Value::String(s.to_string())).ok() } +} - pub fn control_write_max_faults(&self) -> usize { - match self { - Self::None => 0, - _ => 1, +impl AutoCrdt for ConsistencyMode { + const WARN_IF_DIFFERENT: bool = true; +} + +impl ReplicationFactor { + pub fn new(replication_factor: usize) -> Option { + if replication_factor < 1 { + None + } else { + Some(Self(replication_factor)) } } pub fn replication_factor(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay | Self::TwoWayDangerous => 2, - Self::ThreeWay | Self::ThreeWayDegraded | Self::ThreeWayDangerous => 3, + self.0 + } + + pub fn read_quorum(&self, consistency_mode: ConsistencyMode) -> usize { + match consistency_mode { + ConsistencyMode::Dangerous | ConsistencyMode::Degraded => 1, + ConsistencyMode::Consistent => self.replication_factor().div_ceil(2), } } - pub fn read_quorum(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay | Self::TwoWayDangerous => 1, - Self::ThreeWay => 2, - Self::ThreeWayDegraded | Self::ThreeWayDangerous => 1, - } - } - - pub fn write_quorum(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay => 2, - Self::TwoWayDangerous => 1, - Self::ThreeWay | Self::ThreeWayDegraded => 2, - Self::ThreeWayDangerous => 1, + pub fn write_quorum(&self, consistency_mode: ConsistencyMode) -> usize { + match consistency_mode { + ConsistencyMode::Dangerous => 1, + ConsistencyMode::Degraded | ConsistencyMode::Consistent => { + (self.replication_factor() + 1) - self.read_quorum(ConsistencyMode::Consistent) + } } } } + +impl std::convert::From for usize { + fn from(replication_factor: ReplicationFactor) -> usize { + replication_factor.0 + } +} + +pub fn parse_replication_mode( + config: &Config, +) -> Result<(ReplicationFactor, ConsistencyMode), Error> { + match (&config.replication_mode, config.replication_factor, config.consistency_mode.as_str()) { + (Some(replication_mode), None, "consistent") => { + tracing::warn!("Legacy config option replication_mode in use. Please migrate to replication_factor and consistency_mode"); + let parsed_replication_mode = match replication_mode.as_str() { + "1" | "none" => Some((ReplicationFactor(1), ConsistencyMode::Consistent)), + "2" => Some((ReplicationFactor(2), ConsistencyMode::Consistent)), + "2-dangerous" => Some((ReplicationFactor(2), ConsistencyMode::Dangerous)), + "3" => Some((ReplicationFactor(3), ConsistencyMode::Consistent)), + "3-degraded" => Some((ReplicationFactor(3), ConsistencyMode::Degraded)), + "3-dangerous" => Some((ReplicationFactor(3), ConsistencyMode::Dangerous)), + _ => None, + }; + Some(parsed_replication_mode.ok_or_message("Invalid replication_mode in config file.")?) + }, + (None, Some(replication_factor), consistency_mode) => { + let replication_factor = ReplicationFactor::new(replication_factor) + .ok_or_message("Invalid replication_factor in config file.")?; + let consistency_mode = ConsistencyMode::parse(consistency_mode) + .ok_or_message("Invalid consistency_mode in config file.")?; + Some((replication_factor, consistency_mode)) + } + _ => None, + }.ok_or_message("Either the legacy replication_mode or replication_level and consistency_mode can be set, not both.") +} diff --git a/src/rpc/ring.rs b/src/rpc/ring.rs deleted file mode 100644 index 6a2e5c72..00000000 --- a/src/rpc/ring.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Module containing types related to computing nodes which should receive a copy of data blocks -//! and metadata -use std::convert::TryInto; - -use garage_util::data::*; - -use crate::layout::ClusterLayout; - -/// A partition id, which is stored on 16 bits -/// i.e. we have up to 2**16 partitions. -/// (in practice we have exactly 2**PARTITION_BITS partitions) -pub type Partition = u16; - -// TODO: make this constant parametrizable in the config file -// For deployments with many nodes it might make sense to bump -// it up to 10. -// Maximum value : 16 -/// How many bits from the hash are used to make partitions. Higher numbers means more fairness in -/// presence of numerous nodes, but exponentially bigger ring. Max 16 -pub const PARTITION_BITS: usize = 8; - -const PARTITION_MASK_U16: u16 = ((1 << PARTITION_BITS) - 1) << (16 - PARTITION_BITS); - -/// A ring distributing fairly objects to nodes -#[derive(Clone)] -pub struct Ring { - /// The replication factor for this ring - pub replication_factor: usize, - - /// The network configuration used to generate this ring - pub layout: ClusterLayout, - - // Internal order of nodes used to make a more compact representation of the ring - nodes: Vec, - - // The list of entries in the ring - ring: Vec, -} - -// Type to store compactly the id of a node in the system -// Change this to u16 the day we want to have more than 256 nodes in a cluster -pub type CompactNodeType = u8; -pub const MAX_NODE_NUMBER: usize = 256; - -// The maximum number of times an object might get replicated -// This must be at least 3 because Garage supports 3-way replication -// Here we use 6 so that the size of a ring entry is 8 bytes -// (2 bytes partition id, 6 bytes node numbers as u8s) -const MAX_REPLICATION: usize = 6; - -/// An entry in the ring -#[derive(Clone, Debug)] -struct RingEntry { - // The two first bytes of the first hash that goes in this partition - // (the next bytes are zeroes) - hash_prefix: u16, - // The nodes that store this partition, stored as a list of positions in the `nodes` - // field of the Ring structure - // Only items 0 up to ring.replication_factor - 1 are used, others are zeros - nodes_buf: [CompactNodeType; MAX_REPLICATION], -} - -impl Ring { - pub(crate) fn new(layout: ClusterLayout, replication_factor: usize) -> Self { - if replication_factor != layout.replication_factor { - warn!("Could not build ring: replication factor does not match between local configuration and network role assignment."); - return Self::empty(layout, replication_factor); - } - - if layout.ring_assignment_data.len() != replication_factor * (1 << PARTITION_BITS) { - warn!("Could not build ring: network role assignment data has invalid length"); - return Self::empty(layout, replication_factor); - } - - let nodes = layout.node_id_vec.clone(); - let ring = (0..(1 << PARTITION_BITS)) - .map(|i| { - let top = (i as u16) << (16 - PARTITION_BITS); - let mut nodes_buf = [0u8; MAX_REPLICATION]; - nodes_buf[..replication_factor].copy_from_slice( - &layout.ring_assignment_data - [replication_factor * i..replication_factor * (i + 1)], - ); - RingEntry { - hash_prefix: top, - nodes_buf, - } - }) - .collect::>(); - - Self { - replication_factor, - layout, - nodes, - ring, - } - } - - fn empty(layout: ClusterLayout, replication_factor: usize) -> Self { - Self { - replication_factor, - layout, - nodes: vec![], - ring: vec![], - } - } - - /// Get the partition in which data would fall on - pub fn partition_of(&self, position: &Hash) -> Partition { - let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); - top >> (16 - PARTITION_BITS) - } - - /// Get the list of partitions and the first hash of a partition key that would fall in it - pub fn partitions(&self) -> Vec<(Partition, Hash)> { - let mut ret = vec![]; - - for (i, entry) in self.ring.iter().enumerate() { - let mut location = [0u8; 32]; - location[..2].copy_from_slice(&u16::to_be_bytes(entry.hash_prefix)[..]); - ret.push((i as u16, location.into())); - } - if !ret.is_empty() { - assert_eq!(ret[0].1, [0u8; 32].into()); - } - - ret - } - - /// Walk the ring to find the n servers in which data should be replicated - pub fn get_nodes(&self, position: &Hash, n: usize) -> Vec { - if self.ring.len() != 1 << PARTITION_BITS { - warn!("Ring not yet ready, read/writes will be lost!"); - return vec![]; - } - - let partition_idx = self.partition_of(position) as usize; - let partition = &self.ring[partition_idx]; - - let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); - // Check that we haven't messed up our partition table, i.e. that this partition - // table entrey indeed corresponds to the item we are storing - assert_eq!( - partition.hash_prefix & PARTITION_MASK_U16, - top & PARTITION_MASK_U16 - ); - - assert!(n <= self.replication_factor); - partition.nodes_buf[..n] - .iter() - .map(|i| self.nodes[*i as usize]) - .collect::>() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ring_entry_size() { - assert_eq!(std::mem::size_of::(), 8); - } -} diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index e59c372a..2505c2ce 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -1,12 +1,12 @@ //! Contain structs related to making RPCs -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; use std::time::Duration; use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; use futures::stream::StreamExt; use tokio::select; -use tokio::sync::watch; use opentelemetry::KeyValue; use opentelemetry::{ @@ -14,35 +14,36 @@ use opentelemetry::{ Context, }; -pub use netapp::endpoint::{Endpoint, EndpointHandler, StreamingEndpointHandler}; -pub use netapp::message::{ +pub use garage_net::endpoint::{Endpoint, EndpointHandler, StreamingEndpointHandler}; +pub use garage_net::message::{ IntoReq, Message as Rpc, OrderTag, Req, RequestPriority, Resp, PRIO_BACKGROUND, PRIO_HIGH, PRIO_NORMAL, PRIO_SECONDARY, }; -use netapp::peering::fullmesh::FullMeshPeeringStrategy; -pub use netapp::{self, NetApp, NodeID}; +use garage_net::peering::PeeringManager; +pub use garage_net::{self, NetApp, NodeID}; use garage_util::data::*; use garage_util::error::Error; use garage_util::metrics::RecordDuration; +use crate::layout::{LayoutHelper, LayoutVersion}; use crate::metrics::RpcMetrics; -use crate::ring::Ring; // Default RPC timeout = 5 minutes const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); /// Strategy to apply when making RPC -#[derive(Copy, Clone)] -pub struct RequestStrategy { +pub struct RequestStrategy { /// Min number of response to consider the request successful - pub rs_quorum: Option, - /// Should requests be dropped after enough response are received - pub rs_interrupt_after_quorum: bool, + rs_quorum: Option, + /// Send all requests at once + rs_send_all_at_once: Option, /// Request priority - pub rs_priority: RequestPriority, + rs_priority: RequestPriority, /// Custom timeout for this request rs_timeout: Timeout, + /// Data to drop when everything completes + rs_drop_on_complete: T, } #[derive(Copy, Clone)] @@ -52,25 +53,50 @@ enum Timeout { Custom(Duration), } -impl RequestStrategy { +impl Clone for RequestStrategy<()> { + fn clone(&self) -> Self { + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: (), + } + } +} + +impl RequestStrategy<()> { /// Create a RequestStrategy with default timeout and not interrupting when quorum reached pub fn with_priority(prio: RequestPriority) -> Self { RequestStrategy { rs_quorum: None, - rs_interrupt_after_quorum: false, + rs_send_all_at_once: None, rs_priority: prio, rs_timeout: Timeout::Default, + rs_drop_on_complete: (), } } + /// Add an item to be dropped on completion + pub fn with_drop_on_completion(self, drop_on_complete: T) -> RequestStrategy { + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: drop_on_complete, + } + } +} + +impl RequestStrategy { /// Set quorum to be reached for request pub fn with_quorum(mut self, quorum: usize) -> Self { self.rs_quorum = Some(quorum); self } - /// Set if requests can be dropped after quorum has been reached - /// In general true for read requests, and false for write - pub fn interrupt_after_quorum(mut self, interrupt: bool) -> Self { - self.rs_interrupt_after_quorum = interrupt; + /// Set quorum to be reached for request + pub fn send_all_at_once(mut self, value: bool) -> Self { + self.rs_send_all_at_once = Some(value); self } /// Deactivate timeout for this request @@ -83,6 +109,19 @@ impl RequestStrategy { self.rs_timeout = Timeout::Custom(timeout); self } + /// Extract drop_on_complete item + fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) { + ( + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: (), + }, + self.rs_drop_on_complete, + ) + } } #[derive(Clone)] @@ -90,8 +129,8 @@ pub struct RpcHelper(Arc); struct RpcHelperInner { our_node_id: Uuid, - fullmesh: Arc, - ring: watch::Receiver>, + peering: Arc, + layout: Arc>, metrics: RpcMetrics, rpc_timeout: Duration, } @@ -99,16 +138,16 @@ struct RpcHelperInner { impl RpcHelper { pub(crate) fn new( our_node_id: Uuid, - fullmesh: Arc, - ring: watch::Receiver>, + peering: Arc, + layout: Arc>, rpc_timeout: Option, ) -> Self { let metrics = RpcMetrics::new(); Self(Arc::new(RpcHelperInner { our_node_id, - fullmesh, - ring, + peering, + layout, metrics, rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT), })) @@ -123,13 +162,19 @@ impl RpcHelper { endpoint: &Endpoint, to: Uuid, msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result where M: Rpc>, N: IntoReq + Send, H: StreamingEndpointHandler, { + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!("RPC [{}] to {:?}", endpoint.path(), to); + let mut span = tracer.start(span_name); + span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); + span.set_attribute(KeyValue::new("to", format!("{:?}", to))); + let metric_tags = [ KeyValue::new("rpc_endpoint", endpoint.path().to_string()), KeyValue::new("from", format!("{:?}", self.0.our_node_id)), @@ -141,6 +186,7 @@ impl RpcHelper { let node_id = to.into(); let rpc_call = endpoint .call_streaming(&node_id, msg, strat.rs_priority) + .with_context(Context::current_with_span(span)) .record_duration(&self.0.metrics.rpc_duration, &metric_tags); let timeout = async { @@ -176,19 +222,24 @@ impl RpcHelper { endpoint: &Endpoint, to: &[Uuid], msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result)>, Error> where M: Rpc>, N: IntoReq, H: StreamingEndpointHandler, { - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!("RPC [{}] call_many {} nodes", endpoint.path(), to.len()); + let span = tracer.start(span_name); + + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; let resps = join_all( to.iter() - .map(|to| self.call(endpoint, *to, msg.clone(), strat)), + .map(|to| self.call(endpoint, *to, msg.clone(), strat.clone())), ) + .with_context(Context::current_with_span(span)) .await; Ok(to .iter() @@ -201,7 +252,7 @@ impl RpcHelper { &self, endpoint: &Endpoint, msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result)>, Error> where M: Rpc>, @@ -210,7 +261,7 @@ impl RpcHelper { { let to = self .0 - .fullmesh + .peering .get_peer_list() .iter() .map(|p| p.id.into()) @@ -220,12 +271,28 @@ impl RpcHelper { /// Make a RPC call to multiple servers, returning either a Vec of responses, /// or an error if quorum could not be reached due to too many errors + /// + /// If RequestStrategy has send_all_at_once set, then all requests will be + /// sent at once, and `try_call_many` will return as soon as a quorum of + /// responses is achieved, dropping and cancelling the remaining requests. + /// + /// Otherwise, `quorum` requests will be sent at the same time, and if an + /// error response is received, a new request will be sent to replace it. + /// The ordering of nodes to which requests are sent is determined by + /// the `RpcHelper::request_order` function, which takes into account + /// parameters such as node zones and measured ping values. + /// + /// In both cases, the basic contract of this function is that even in the + /// absence of failures, the RPC call might not be driven to completion + /// on all of the specified nodes. It is therefore unfit for broadcast + /// write operations where we expect all nodes to successfully store + /// the written date. pub async fn try_call_many( &self, endpoint: &Arc>, to: &[Uuid], msg: N, - strategy: RequestStrategy, + strategy: RequestStrategy<()>, ) -> Result, Error> where M: Rpc> + 'static, @@ -236,36 +303,29 @@ impl RpcHelper { let quorum = strategy.rs_quorum.unwrap_or(to.len()); let tracer = opentelemetry::global::tracer("garage"); - let span_name = if strategy.rs_interrupt_after_quorum { - format!("RPC {} to {} of {}", endpoint.path(), quorum, to.len()) - } else { - format!( - "RPC {} to {} (quorum {})", - endpoint.path(), - to.len(), - quorum - ) - }; + let span_name = format!( + "RPC [{}] try_call_many (quorum {}/{})", + endpoint.path(), + quorum, + to.len() + ); + let mut span = tracer.start(span_name); span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); - span.set_attribute(KeyValue::new( - "interrupt_after_quorum", - strategy.rs_interrupt_after_quorum.to_string(), - )); - self.try_call_many_internal(endpoint, to, msg, strategy, quorum) + self.try_call_many_inner(endpoint, to, msg, strategy, quorum) .with_context(Context::current_with_span(span)) .await } - async fn try_call_many_internal( + async fn try_call_many_inner( &self, endpoint: &Arc>, to: &[Uuid], msg: N, - strategy: RequestStrategy, + strategy: RequestStrategy<()>, quorum: usize, ) -> Result, Error> where @@ -274,110 +334,65 @@ impl RpcHelper { H: StreamingEndpointHandler + 'static, S: Send + 'static, { - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + // Once quorum is reached, other requests don't matter. + // What we do here is only send the required number of requests + // to reach a quorum, priorizing nodes with the lowest latency. + // When there are errors, we start new requests to compensate. + + // TODO: this could be made more aggressive, e.g. if after 2x the + // average ping of a given request, the response is not yet received, + // preemptively send an additional request to any remaining nodes. + + // Reorder requests to priorize closeness / low latency + let request_order = + self.request_order(&self.0.layout.read().unwrap().current(), to.iter().copied()); + let send_all_at_once = strategy.rs_send_all_at_once.unwrap_or(false); // Build future for each request // They are not started now: they are added below in a FuturesUnordered // object that will take care of polling them (see below) - let requests = to.iter().cloned().map(|to| { + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; + let mut requests = request_order.into_iter().map(|to| { let self2 = self.clone(); let msg = msg.clone(); let endpoint2 = endpoint.clone(); - (to, async move { - self2.call(&endpoint2, to, msg, strategy).await - }) + let strategy = strategy.clone(); + async move { self2.call(&endpoint2, to, msg, strategy).await } }); // Vectors in which success results and errors will be collected let mut successes = vec![]; let mut errors = vec![]; - if strategy.rs_interrupt_after_quorum { - // Case 1: once quorum is reached, other requests don't matter. - // What we do here is only send the required number of requests - // to reach a quorum, priorizing nodes with the lowest latency. - // When there are errors, we start new requests to compensate. + // resp_stream will contain all of the requests that are currently in flight. + // (for the moment none, they will be added in the loop below) + let mut resp_stream = FuturesUnordered::new(); - // Reorder requests to priorize closeness / low latency - let request_order = self.request_order(to); - let mut ord_requests = vec![(); request_order.len()] - .into_iter() - .map(|_| None) - .collect::>(); - for (to, fut) in requests { - let i = request_order.iter().position(|x| *x == to).unwrap(); - ord_requests[i] = Some((to, fut)); - } - - // Make an iterator to take requests in their sorted order - let mut requests = ord_requests.into_iter().map(Option::unwrap); - - // resp_stream will contain all of the requests that are currently in flight. - // (for the moment none, they will be added in the loop below) - let mut resp_stream = FuturesUnordered::new(); - - // Do some requests and collect results - 'request_loop: while successes.len() < quorum { - // If the current set of requests that are running is not enough to possibly - // reach quorum, start some new requests. - while successes.len() + resp_stream.len() < quorum { - if let Some((req_to, fut)) = requests.next() { - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer.start(format!("RPC to {:?}", req_to)); - resp_stream.push(tokio::spawn( - fut.with_context(Context::current_with_span(span)), - )); - } else { - // If we have no request to add, we know that we won't ever - // reach quorum: bail out now. - break 'request_loop; - } - } - assert!(!resp_stream.is_empty()); // because of loop invariants - - // Wait for one request to terminate - match resp_stream.next().await.unwrap().unwrap() { - Ok(msg) => { - successes.push(msg); - } - Err(e) => { - errors.push(e); - } - } - } - } else { - // Case 2: all of the requests need to be sent in all cases, - // and need to terminate. (this is the case for writes that - // must be spread to n nodes) - // Just start all the requests in parallel and return as soon - // as the quorum is reached. - let mut resp_stream = requests - .map(|(_, fut)| fut) - .collect::>(); - - while let Some(resp) = resp_stream.next().await { - match resp { - Ok(msg) => { - successes.push(msg); - if successes.len() >= quorum { - break; - } - } - Err(e) => { - errors.push(e); - } + // Do some requests and collect results + while successes.len() < quorum { + // If the current set of requests that are running is not enough to possibly + // reach quorum, start some new requests. + while send_all_at_once || successes.len() + resp_stream.len() < quorum { + if let Some(fut) = requests.next() { + resp_stream.push(fut) + } else { + break; } } - if !resp_stream.is_empty() { - // Continue remaining requests in background. - // Note: these requests can get interrupted on process shutdown, - // we must not count on them being executed for certain. - // For all background things that have to happen with certainty, - // they have to be put in a proper queue that is persisted to disk. - tokio::spawn(async move { - resp_stream.collect::>>().await; - }); + if successes.len() + resp_stream.len() < quorum { + // We know we won't ever reach quorum + break; + } + + // Wait for one request to terminate + match resp_stream.next().await.unwrap() { + Ok(msg) => { + successes.push(msg); + } + Err(e) => { + errors.push(e); + } } } @@ -385,19 +400,233 @@ impl RpcHelper { Ok(successes) } else { let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) + Err(Error::Quorum( + quorum, + None, + successes.len(), + to.len(), + errors, + )) } } - pub fn request_order(&self, nodes: &[Uuid]) -> Vec { - // Retrieve some status variables that we will use to sort requests - let peer_list = self.0.fullmesh.get_peer_list(); - let ring: Arc = self.0.ring.borrow().clone(); - let our_zone = match ring.layout.node_role(&self.0.our_node_id) { - Some(pc) => &pc.zone, - None => "", + /// Make a RPC call to multiple servers, returning either a Vec of responses, + /// or an error if quorum could not be reached due to too many errors + /// + /// Contrary to try_call_many, this function is especially made for broadcast + /// write operations. In particular: + /// + /// - The request are sent to all specified nodes as soon as `try_write_many_sets` + /// is invoked. + /// + /// - When `try_write_many_sets` returns, all remaining requests that haven't + /// completed move to a background task so that they have a chance to + /// complete successfully if there are no failures. + /// + /// In addition, the nodes to which requests should be sent are divided in + /// "quorum sets", and `try_write_many_sets` only returns once a quorum + /// has been validated in each set. This is used in the case of cluster layout + /// changes, where data has to be written both in the old layout and in the + /// new one as long as all nodes have not successfully tranisitionned and + /// moved all data to the new layout. + pub async fn try_write_many_sets( + &self, + endpoint: &Arc>, + to_sets: &[Vec], + msg: N, + strategy: RequestStrategy, + ) -> Result, Error> + where + M: Rpc> + 'static, + N: IntoReq, + H: StreamingEndpointHandler + 'static, + S: Send + 'static, + T: Send + 'static, + { + let quorum = strategy + .rs_quorum + .expect("internal error: missing quorum value in try_write_many_sets"); + + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!( + "RPC [{}] try_write_many_sets (quorum {} in {} sets)", + endpoint.path(), + quorum, + to_sets.len() + ); + + let mut span = tracer.start(span_name); + span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); + span.set_attribute(KeyValue::new("to", format!("{:?}", to_sets))); + span.set_attribute(KeyValue::new("quorum", quorum as i64)); + + self.try_write_many_sets_inner(endpoint, to_sets, msg, strategy, quorum) + .with_context(Context::current_with_span(span)) + .await + } + + async fn try_write_many_sets_inner( + &self, + endpoint: &Arc>, + to_sets: &[Vec], + msg: N, + strategy: RequestStrategy, + quorum: usize, + ) -> Result, Error> + where + M: Rpc> + 'static, + N: IntoReq, + H: StreamingEndpointHandler + 'static, + S: Send + 'static, + T: Send + 'static, + { + // Peers may appear in many quorum sets. Here, build a list of peers, + // mapping to the index of the quorum sets in which they appear. + let mut result_tracker = QuorumSetResultTracker::new(to_sets, quorum); + + let (strategy, drop_on_complete) = strategy.extract_drop_on_complete(); + + // Send one request to each peer of the quorum sets + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; + let requests = result_tracker.nodes.keys().map(|peer| { + let self2 = self.clone(); + let msg = msg.clone(); + let endpoint2 = endpoint.clone(); + let to = *peer; + let strategy = strategy.clone(); + async move { (to, self2.call(&endpoint2, to, msg, strategy).await) } + }); + let mut resp_stream = requests.collect::>(); + + // Drive requests to completion + while let Some((node, resp)) = resp_stream.next().await { + // Store the response in the correct vector and increment the + // appropriate counters + result_tracker.register_result(node, resp); + + // If we have a quorum of ok in all quorum sets, then it's a success! + if result_tracker.all_quorums_ok() { + // Continue all other requests in background + tokio::spawn(async move { + resp_stream.collect::)>>().await; + drop(drop_on_complete); + }); + + return Ok(result_tracker.success_values()); + } + + // If there is a quorum set for which too many errors were received, + // we know it's impossible to get a quorum, so return immediately. + if result_tracker.too_many_failures() { + break; + } + } + + // At this point, there is no quorum and we know that a quorum + // will never be achieved. Currently, we drop all remaining requests. + // Should we still move them to background so that they can continue + // for non-failed nodes? Not doing so has no impact on correctness, + // but it means that more cancellation messages will be sent. Idk. + // (When an in-progress request future is dropped, Netapp automatically + // sends a cancellation message to the remote node to inform it that + // the result is no longer needed. In turn, if the remote node receives + // the cancellation message in time, it interrupts the task of the + // running request handler.) + + // Failure, could not get quorum + Err(result_tracker.quorum_error()) + } + + // ---- functions not related to MAKING RPCs, but just determining to what nodes + // they should be made and in which order ---- + + /// Determine to what nodes, and in what order, requests to read a data block + /// should be sent. All nodes in the Vec returned by this function are tried + /// one by one until there is one that returns the block (in block/manager.rs). + /// + /// We want to have the best chance of finding the block in as few requests + /// as possible, and we want to avoid nodes that answer slowly. + /// + /// Note that when there are several active layout versions, the block might + /// be stored only by nodes of the latest version (in case of a block that was + /// written after the layout change), or only by nodes of the oldest active + /// version (for all blocks that were written before). So we have to try nodes + /// of all layout versions. We also want to try nodes of all layout versions + /// fast, so as to optimize the chance of finding the block fast. + /// + /// Therefore, the strategy is the following: + /// + /// 1. ask first all nodes of all currently active layout versions + /// -> ask the preferred node in all layout versions (older to newer), + /// then the second preferred onde in all verions, etc. + /// -> we start by the oldest active layout version first, because a majority + /// of blocks will have been saved before the layout change + /// 2. ask all nodes of historical layout versions, for blocks which have not + /// yet been transferred to their new storage nodes + /// + /// The preference order, for each layout version, is given by `request_order`, + /// based on factors such as nodes being in the same datacenter, + /// having low ping, etc. + pub fn block_read_nodes_of(&self, position: &Hash, rpc_helper: &RpcHelper) -> Vec { + let layout = self.0.layout.read().unwrap(); + + // Compute, for each layout version, the set of nodes that might store + // the block, and put them in their preferred order as of `request_order`. + let mut vernodes = layout.versions().iter().map(|ver| { + let nodes = ver.nodes_of(position, ver.replication_factor); + rpc_helper.request_order(layout.current(), nodes) + }); + + let mut ret = if layout.versions().len() == 1 { + // If we have only one active layout version, then these are the + // only nodes we ask in step 1 + vernodes.next().unwrap() + } else { + let vernodes = vernodes.collect::>(); + + let mut nodes = Vec::::with_capacity(12); + for i in 0..layout.current().replication_factor { + for vn in vernodes.iter() { + if let Some(n) = vn.get(i) { + if !nodes.contains(&n) { + if *n == self.0.our_node_id { + // it's always fast (almost free) to ask locally, + // so always put that as first choice + nodes.insert(0, *n); + } else { + nodes.push(*n); + } + } + } + } + } + + nodes }; + // Second step: add nodes of older layout versions + let old_ver_iter = layout.inner().old_versions.iter().rev(); + for ver in old_ver_iter { + let nodes = ver.nodes_of(position, ver.replication_factor); + for node in rpc_helper.request_order(layout.current(), nodes) { + if !ret.contains(&node) { + ret.push(node); + } + } + } + + ret + } + + fn request_order( + &self, + layout: &LayoutVersion, + nodes: impl Iterator, + ) -> Vec { + // Retrieve some status variables that we will use to sort requests + let peer_list = self.0.peering.get_peer_list(); + let our_zone = layout.get_node_zone(&self.0.our_node_id).unwrap_or(""); + // Augment requests with some information used to sort them. // The tuples are as follows: // (is another node?, is another zone?, latency, node ID, request future) @@ -405,22 +634,18 @@ impl RpcHelper { // By sorting this vec, we priorize ourself, then nodes in the same zone, // and within a same zone we priorize nodes with the lowest latency. let mut nodes = nodes - .iter() .map(|to| { - let peer_zone = match ring.layout.node_role(to) { - Some(pc) => &pc.zone, - None => "", - }; + let peer_zone = layout.get_node_zone(&to).unwrap_or(""); let peer_avg_ping = peer_list .iter() .find(|x| x.id.as_ref() == to.as_slice()) .and_then(|pi| pi.avg_ping) .unwrap_or_else(|| Duration::from_secs(10)); ( - *to != self.0.our_node_id, + to != self.0.our_node_id, peer_zone != our_zone, peer_avg_ping, - *to, + to, ) }) .collect::>(); @@ -434,3 +659,108 @@ impl RpcHelper { .collect::>() } } + +// ------- utility for tracking successes/errors among write sets -------- + +pub struct QuorumSetResultTracker { + /// The set of nodes and the index of the quorum sets they belong to + pub nodes: HashMap>, + /// The quorum value, i.e. number of success responses to await in each set + pub quorum: usize, + + /// The success responses received + pub successes: Vec<(Uuid, S)>, + /// The error responses received + pub failures: Vec<(Uuid, E)>, + + /// The counters for successes in each set + pub success_counters: Box<[usize]>, + /// The counters for failures in each set + pub failure_counters: Box<[usize]>, + /// The total number of nodes in each set + pub set_lens: Box<[usize]>, +} + +impl QuorumSetResultTracker +where + E: std::fmt::Display, +{ + pub fn new(sets: &[A], quorum: usize) -> Self + where + A: AsRef<[Uuid]>, + { + let mut nodes = HashMap::>::new(); + for (i, set) in sets.iter().enumerate() { + for node in set.as_ref().iter() { + nodes.entry(*node).or_default().push(i); + } + } + + let num_nodes = nodes.len(); + Self { + nodes, + quorum, + successes: Vec::with_capacity(num_nodes), + failures: vec![], + success_counters: vec![0; sets.len()].into_boxed_slice(), + failure_counters: vec![0; sets.len()].into_boxed_slice(), + set_lens: sets + .iter() + .map(|x| x.as_ref().len()) + .collect::>() + .into_boxed_slice(), + } + } + + pub fn register_result(&mut self, node: Uuid, result: Result) { + match result { + Ok(s) => { + self.successes.push((node, s)); + for set in self.nodes.get(&node).unwrap().iter() { + self.success_counters[*set] += 1; + } + } + Err(e) => { + self.failures.push((node, e)); + for set in self.nodes.get(&node).unwrap().iter() { + self.failure_counters[*set] += 1; + } + } + } + } + + pub fn all_quorums_ok(&self) -> bool { + self.success_counters + .iter() + .all(|ok_cnt| *ok_cnt >= self.quorum) + } + + pub fn too_many_failures(&self) -> bool { + self.failure_counters + .iter() + .zip(self.set_lens.iter()) + .any(|(err_cnt, set_len)| *err_cnt + self.quorum > *set_len) + } + + pub fn success_values(self) -> Vec { + self.successes + .into_iter() + .map(|(_, x)| x) + .collect::>() + } + + pub fn quorum_error(self) -> Error { + let errors = self + .failures + .iter() + .map(|(n, e)| format!("{:?}: {}", n, e)) + .collect::>(); + Error::Quorum( + self.quorum, + Some(self.set_lens.len()), + self.successes.len(), + self.nodes.len(), + errors, + ) + } +} diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 7fc3c20c..2a52ae5d 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -1,26 +1,23 @@ //! Module containing structs related to membership management -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::{Read, Write}; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; -use std::sync::atomic::Ordering; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, RwLockReadGuard}; use std::time::{Duration, Instant}; -use arc_swap::ArcSwap; -use async_trait::async_trait; -use futures::{join, select}; -use futures_util::future::*; +use arc_swap::ArcSwapOption; +use futures::join; use serde::{Deserialize, Serialize}; use sodiumoxide::crypto::sign::ed25519; -use tokio::sync::watch; -use tokio::sync::Mutex; +use tokio::select; +use tokio::sync::{watch, Notify}; -use netapp::endpoint::{Endpoint, EndpointHandler}; -use netapp::message::*; -use netapp::peering::fullmesh::FullMeshPeeringStrategy; -use netapp::util::parse_and_resolve_peer_addr_async; -use netapp::{NetApp, NetworkKey, NodeID, NodeKey}; +use garage_net::endpoint::{Endpoint, EndpointHandler}; +use garage_net::message::*; +use garage_net::peering::{PeerConnState, PeeringManager}; +use garage_net::util::parse_and_resolve_peer_addr_async; +use garage_net::{NetApp, NetworkKey, NodeID, NodeKey}; #[cfg(feature = "kubernetes-discovery")] use garage_util::config::KubernetesDiscoveryConfig; @@ -34,9 +31,10 @@ use garage_util::time::*; use crate::consul::ConsulDiscovery; #[cfg(feature = "kubernetes-discovery")] use crate::kubernetes::*; -use crate::layout::*; +use crate::layout::{ + self, manager::LayoutManager, LayoutHelper, LayoutHistory, NodeRoleV, RpcLayoutDigest, +}; use crate::replication_mode::*; -use crate::ring::*; use crate::rpc_helper::*; use crate::system_metrics::*; @@ -47,29 +45,34 @@ const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); /// Version tag used for version check upon Netapp connection. /// Cluster nodes with different version tags are deemed /// incompatible and will refuse to connect. -pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008 +pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650010; // garage 0x0010 (1.0) /// RPC endpoint used for calls related to membership -pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; +pub const SYSTEM_RPC_PATH: &str = "garage_rpc/system.rs/SystemRpc"; /// RPC messages related to membership #[derive(Debug, Serialize, Deserialize, Clone)] pub enum SystemRpc { - /// Response to successfull advertisements + /// Response to successful advertisements Ok, - /// Request to connect to a specific node (in @: format) + /// Request to connect to a specific node (in @: format, pubkey = full-length node ID) Connect(String), - /// Ask other node its cluster layout. Answered with AdvertiseClusterLayout - PullClusterLayout, /// Advertise Garage status. Answered with another AdvertiseStatus. /// Exchanged with every node on a regular basis. AdvertiseStatus(NodeStatus), - /// Advertisement of cluster layout. Sent spontanously or in response to PullClusterLayout - AdvertiseClusterLayout(ClusterLayout), /// Get known nodes states GetKnownNodes, /// Return known nodes ReturnKnownNodes(Vec), + + /// Ask other node its cluster layout. Answered with AdvertiseClusterLayout + PullClusterLayout, + /// Advertisement of cluster layout. Sent spontanously or in response to PullClusterLayout + AdvertiseClusterLayout(LayoutHistory), + /// Ask other node its cluster layout update trackers. + PullClusterLayoutTrackers, + /// Advertisement of cluster layout update trackers. + AdvertiseClusterLayoutTrackers(layout::UpdateTrackers), } impl Rpc for SystemRpc { @@ -85,20 +88,17 @@ pub struct System { /// The id of this node pub id: Uuid, - persist_cluster_layout: Persister, persist_peer_list: Persister, - local_status: ArcSwap, + pub(crate) local_status: RwLock, node_status: RwLock>, pub netapp: Arc, - fullmesh: Arc, - pub rpc: RpcHelper, + peering: Arc, - system_endpoint: Arc>, + pub(crate) system_endpoint: Arc>, rpc_listen_addr: SocketAddr, - #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))] rpc_public_addr: Option, bootstrap_peers: Vec, @@ -107,14 +107,11 @@ pub struct System { #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: Option, - metrics: SystemMetrics, + pub layout_manager: Arc, - replication_mode: ReplicationMode, - replication_factor: usize, + metrics: ArcSwapOption, - /// The ring - pub ring: watch::Receiver>, - update_ring: Mutex>>, + pub(crate) replication_factor: ReplicationFactor, /// Path to metadata directory pub metadata_dir: PathBuf, @@ -125,14 +122,13 @@ pub struct System { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NodeStatus { /// Hostname of the node - pub hostname: String, + pub hostname: Option, /// Replication factor configured on the node pub replication_factor: usize, - /// Cluster layout version - pub cluster_layout_version: u64, - /// Hash of cluster layout staging data - pub cluster_layout_staging_hash: Hash, + + /// Cluster layout digest + pub layout_digest: RpcLayoutDigest, /// Disk usage on partition containing metadata directory (tuple: `(avail, total)`) #[serde(default)] @@ -145,7 +141,7 @@ pub struct NodeStatus { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct KnownNodeInfo { pub id: Uuid, - pub addr: SocketAddr, + pub addr: Option, pub is_up: bool, pub last_seen_secs_ago: Option, pub status: NodeStatus, @@ -171,11 +167,11 @@ pub struct ClusterHealth { pub partitions_all_ok: usize, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum ClusterHealthStatus { /// All nodes are available Healthy, - /// Some storage nodes are unavailable, but quorum is stil + /// Some storage nodes are unavailable, but quorum is still /// achieved for all partitions Degraded, /// Quorum is not available for some partitions @@ -245,11 +241,11 @@ impl System { /// Create this node's membership manager pub fn new( network_key: NetworkKey, - replication_mode: ReplicationMode, + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, config: &Config, ) -> Result, Error> { - let replication_factor = replication_mode.replication_factor(); - + // ---- setup netapp RPC protocol ---- let node_key = gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID"); info!( @@ -257,82 +253,39 @@ impl System { hex::encode(&node_key.public_key()[..8]) ); - let persist_cluster_layout: Persister = - Persister::new(&config.metadata_dir, "cluster_layout"); - let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); + let bind_outgoing_to = Some(config) + .filter(|x| x.rpc_bind_outgoing) + .map(|x| x.rpc_bind_addr.ip()); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key, bind_outgoing_to); + let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into()); - let cluster_layout = match persist_cluster_layout.load() { - Ok(x) => { - if x.replication_factor != replication_factor { - return Err(Error::Message(format!( - "Prevous cluster layout has replication factor {}, which is different than the one specified in the config file ({}). The previous cluster layout can be purged, if you know what you are doing, simply by deleting the `cluster_layout` file in your metadata directory.", - x.replication_factor, - replication_factor - ))); - } - x - } - Err(e) => { - info!( - "No valid previous cluster layout stored ({}), starting fresh.", - e - ); - ClusterLayout::new(replication_factor) - } - }; - - let metrics = SystemMetrics::new(replication_factor); - - let mut local_status = NodeStatus::initial(replication_factor, &cluster_layout); - local_status.update_disk_usage(&config.metadata_dir, &config.data_dir, &metrics); - - let ring = Ring::new(cluster_layout, replication_factor); - let (update_ring, ring) = watch::channel(Arc::new(ring)); - - let rpc_public_addr = match &config.rpc_public_addr { - Some(a_str) => { - use std::net::ToSocketAddrs; - match a_str.to_socket_addrs() { - Err(e) => { - error!( - "Cannot resolve rpc_public_addr {} from config file: {}.", - a_str, e - ); - None - } - Ok(a) => { - let a = a.collect::>(); - if a.is_empty() { - error!("rpc_public_addr {} resolve to no known IP address", a_str); - } - if a.len() > 1 { - warn!("Multiple possible resolutions for rpc_public_addr: {:?}. Taking the first one.", a); - } - a.into_iter().next() - } - } - } - None => { - let addr = - get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); - if let Some(a) = addr { - warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); - } - addr - } - }; + // ---- setup netapp public listener and full mesh peering strategy ---- + let rpc_public_addr = get_rpc_public_addr(config); if rpc_public_addr.is_none() { warn!("This Garage node does not know its publicly reachable RPC address, this might hamper intra-cluster communication."); } - let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key); - let fullmesh = FullMeshPeeringStrategy::new(netapp.clone(), vec![], rpc_public_addr); + let peering = PeeringManager::new(netapp.clone(), vec![], rpc_public_addr); if let Some(ping_timeout) = config.rpc_ping_timeout_msec { - fullmesh.set_ping_timeout_millis(ping_timeout); + peering.set_ping_timeout_millis(ping_timeout); } - let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into()); + let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); + // ---- setup cluster layout and layout manager ---- + let layout_manager = LayoutManager::new( + config, + netapp.id, + system_endpoint.clone(), + peering.clone(), + replication_factor, + consistency_mode, + )?; + + let mut local_status = NodeStatus::initial(replication_factor, &layout_manager); + local_status.update_disk_usage(&config.metadata_dir, &config.data_dir); + + // ---- if enabled, set up additional peer discovery methods ---- #[cfg(feature = "consul-discovery")] let consul_discovery = match &config.consul_discovery { Some(cfg) => Some( @@ -351,66 +304,87 @@ impl System { warn!("Kubernetes discovery is not enabled in this build."); } + // ---- almost done ---- let sys = Arc::new(System { id: netapp.id.into(), - persist_cluster_layout, persist_peer_list, - local_status: ArcSwap::new(Arc::new(local_status)), + local_status: RwLock::new(local_status), node_status: RwLock::new(HashMap::new()), netapp: netapp.clone(), - fullmesh: fullmesh.clone(), - rpc: RpcHelper::new( - netapp.id.into(), - fullmesh, - ring.clone(), - config.rpc_timeout_msec.map(Duration::from_millis), - ), + peering: peering.clone(), system_endpoint, - replication_mode, replication_factor, rpc_listen_addr: config.rpc_bind_addr, - #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))] rpc_public_addr, bootstrap_peers: config.bootstrap_peers.clone(), #[cfg(feature = "consul-discovery")] consul_discovery, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: config.kubernetes_discovery.clone(), - metrics, + layout_manager, + metrics: ArcSwapOption::new(None), - ring, - update_ring: Mutex::new(update_ring), metadata_dir: config.metadata_dir.clone(), data_dir: config.data_dir.clone(), }); + sys.system_endpoint.set_handler(sys.clone()); + + let metrics = SystemMetrics::new(sys.clone()); + sys.metrics.store(Some(Arc::new(metrics))); + Ok(sys) } - /// Perform bootstraping, starting the ping loop + /// Perform bootstrapping, starting the ping loop pub async fn run(self: Arc, must_exit: watch::Receiver) { join!( - self.netapp - .clone() - .listen(self.rpc_listen_addr, None, must_exit.clone()), - self.fullmesh.clone().run(must_exit.clone()), + self.netapp.clone().listen( + self.rpc_listen_addr, + self.rpc_public_addr, + must_exit.clone() + ), + self.peering.clone().run(must_exit.clone()), self.discovery_loop(must_exit.clone()), self.status_exchange_loop(must_exit.clone()), ); } + pub fn cleanup(&self) { + // Break reference cycle + self.metrics.store(None); + } + + // ---- Public utilities / accessors ---- + + pub fn cluster_layout(&self) -> RwLockReadGuard<'_, LayoutHelper> { + self.layout_manager.layout() + } + + pub fn layout_notify(&self) -> Arc { + self.layout_manager.change_notify.clone() + } + + pub fn rpc_helper(&self) -> &RpcHelper { + &self.layout_manager.rpc_helper + } + // ---- Administrative operations (directly available and // also available through RPC) ---- pub fn get_known_nodes(&self) -> Vec { let node_status = self.node_status.read().unwrap(); let known_nodes = self - .fullmesh + .peering .get_peer_list() .iter() .map(|n| KnownNodeInfo { id: n.id.into(), - addr: n.addr, + addr: match n.state { + PeerConnState::Ourself => self.rpc_public_addr, + PeerConnState::Connected { addr } => Some(addr), + _ => None, + }, is_up: n.is_up(), last_seen_secs_ago: n .last_seen @@ -425,18 +399,6 @@ impl System { known_nodes } - pub fn get_cluster_layout(&self) -> ClusterLayout { - self.ring.borrow().layout.clone() - } - - pub async fn update_cluster_layout( - self: &Arc, - layout: &ClusterLayout, - ) -> Result<(), Error> { - self.handle_advertise_cluster_layout(layout).await?; - Ok(()) - } - pub async fn connect(&self, node: &str) -> Result<(), Error> { let (pubkey, addrs) = parse_and_resolve_peer_addr_async(node) .await @@ -466,47 +428,65 @@ impl System { } pub fn health(&self) -> ClusterHealth { - let ring: Arc<_> = self.ring.borrow().clone(); - let quorum = self.replication_mode.write_quorum(); - let replication_factor = self.replication_factor; + let quorum = self + .replication_factor + .write_quorum(ConsistencyMode::Consistent); + // Gather information about running nodes. + // Technically, `nodes` contains currently running nodes, as well + // as nodes that this Garage process has been connected to at least + // once since it started. let nodes = self .get_known_nodes() .into_iter() .map(|n| (n.id, n)) .collect::>(); let connected_nodes = nodes.iter().filter(|(_, n)| n.is_up).count(); + let node_up = |x: &Uuid| nodes.get(x).map(|n| n.is_up).unwrap_or(false); - let storage_nodes = ring - .layout - .roles - .items() - .iter() - .filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some())) - .collect::>(); - let storage_nodes_ok = storage_nodes - .iter() - .filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false)) - .count(); + // Acquire a rwlock read-lock to the current cluster layout + let layout = self.cluster_layout(); - let partitions = ring.partitions(); - let partitions_n_up = partitions - .iter() - .map(|(_, h)| { - let pn = ring.get_nodes(h, ring.replication_factor); - pn.iter() - .filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false)) - .count() - }) - .collect::>(); - let partitions_all_ok = partitions_n_up - .iter() - .filter(|c| **c == replication_factor) - .count(); - let partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count(); + // Obtain information about nodes that have a role as storage nodes + // in one of the active layout versions + let mut storage_nodes = HashSet::::with_capacity(16); + for ver in layout.versions().iter() { + storage_nodes.extend( + ver.roles + .items() + .iter() + .filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some())) + .map(|(n, _, _)| *n), + ) + } + let storage_nodes_ok = storage_nodes.iter().filter(|x| node_up(x)).count(); + // Determine the number of partitions that have: + // - a quorum of up nodes for all write sets (i.e. are available) + // - for which all nodes in all write sets are up (i.e. are fully healthy) + let partitions = layout.current().partitions().collect::>(); + let mut partitions_quorum = 0; + let mut partitions_all_ok = 0; + for (_, hash) in partitions.iter() { + let mut write_sets = layout + .versions() + .iter() + .map(|x| x.nodes_of(hash, x.replication_factor)); + let has_quorum = write_sets + .clone() + .all(|set| set.filter(|x| node_up(x)).count() >= quorum); + let all_ok = write_sets.all(|mut set| set.all(|x| node_up(&x))); + if has_quorum { + partitions_quorum += 1; + } + if all_ok { + partitions_all_ok += 1; + } + } + + // Determine overall cluster status let status = - if partitions_quorum == partitions.len() && storage_nodes_ok == storage_nodes.len() { + if partitions_all_ok == partitions.len() && storage_nodes_ok == storage_nodes.len() { ClusterHealthStatus::Healthy } else if partitions_quorum == partitions.len() { ClusterHealthStatus::Degraded @@ -543,12 +523,9 @@ impl System { } }; + let hostname = self.local_status.read().unwrap().hostname.clone().unwrap(); if let Err(e) = c - .publish_consul_service( - self.netapp.id, - &self.local_status.load_full().hostname, - rpc_public_addr, - ) + .publish_consul_service(self.netapp.id, &hostname, rpc_public_addr) .await { error!("Error while publishing Consul service: {}", e); @@ -570,38 +547,17 @@ impl System { } }; - if let Err(e) = publish_kubernetes_node( - k, - self.netapp.id, - &self.local_status.load_full().hostname, - rpc_public_addr, - ) - .await + let hostname = self.local_status.read().unwrap().hostname.clone().unwrap(); + if let Err(e) = publish_kubernetes_node(k, self.netapp.id, &hostname, rpc_public_addr).await { error!("Error while publishing node to Kubernetes: {}", e); } } - /// Save network configuration to disc - async fn save_cluster_layout(&self) -> Result<(), Error> { - let ring: Arc = self.ring.borrow().clone(); - self.persist_cluster_layout - .save_async(&ring.layout) - .await - .expect("Cannot save current cluster layout"); - Ok(()) - } - fn update_local_status(&self) { - let mut new_si: NodeStatus = self.local_status.load().as_ref().clone(); - - let ring = self.ring.borrow(); - new_si.cluster_layout_version = ring.layout.version; - new_si.cluster_layout_staging_hash = ring.layout.staging_hash; - - new_si.update_disk_usage(&self.metadata_dir, &self.data_dir, &self.metrics); - - self.local_status.swap(Arc::new(new_si)); + let mut local_status = self.local_status.write().unwrap(); + local_status.layout_digest = self.layout_manager.layout().digest(); + local_status.update_disk_usage(&self.metadata_dir, &self.data_dir); } // --- RPC HANDLERS --- @@ -611,11 +567,6 @@ impl System { Ok(SystemRpc::Ok) } - fn handle_pull_cluster_layout(&self) -> SystemRpc { - let ring = self.ring.borrow().clone(); - SystemRpc::AdvertiseClusterLayout(ring.layout.clone()) - } - fn handle_get_known_nodes(&self) -> SystemRpc { let known_nodes = self.get_known_nodes(); SystemRpc::ReturnKnownNodes(known_nodes) @@ -626,7 +577,7 @@ impl System { from: Uuid, info: &NodeStatus, ) -> Result { - let local_info = self.local_status.load(); + let local_info = self.local_status.read().unwrap(); if local_info.replication_factor < info.replication_factor { error!("Some node have a higher replication factor ({}) than this one ({}). This is not supported and will lead to data corruption. Shutting down for safety.", @@ -635,11 +586,10 @@ impl System { std::process::exit(1); } - if info.cluster_layout_version > local_info.cluster_layout_version - || info.cluster_layout_staging_hash != local_info.cluster_layout_staging_hash - { - tokio::spawn(self.clone().pull_cluster_layout(from)); - } + self.layout_manager + .handle_advertise_status(from, &info.layout_digest); + + drop(local_info); self.node_status .write() @@ -649,90 +599,44 @@ impl System { Ok(SystemRpc::Ok) } - async fn handle_advertise_cluster_layout( - self: &Arc, - adv: &ClusterLayout, - ) -> Result { - if adv.replication_factor != self.replication_factor { - let msg = format!( - "Received a cluster layout from another node with replication factor {}, which is different from what we have in our configuration ({}). Discarding the cluster layout we received.", - adv.replication_factor, - self.replication_factor - ); - error!("{}", msg); - return Err(Error::Message(msg)); - } - - let update_ring = self.update_ring.lock().await; - let mut layout: ClusterLayout = self.ring.borrow().layout.clone(); - - let prev_layout_check = layout.check().is_ok(); - if layout.merge(adv) { - if prev_layout_check && layout.check().is_err() { - error!("New cluster layout is invalid, discarding."); - return Err(Error::Message( - "New cluster layout is invalid, discarding.".into(), - )); - } - - let ring = Ring::new(layout.clone(), self.replication_factor); - update_ring.send(Arc::new(ring))?; - drop(update_ring); - - let self2 = self.clone(); - tokio::spawn(async move { - if let Err(e) = self2 - .rpc - .broadcast( - &self2.system_endpoint, - SystemRpc::AdvertiseClusterLayout(layout), - RequestStrategy::with_priority(PRIO_HIGH), - ) - .await - { - warn!("Error while broadcasting new cluster layout: {}", e); - } - }); - - self.save_cluster_layout().await?; - } - - Ok(SystemRpc::Ok) - } - async fn status_exchange_loop(&self, mut stop_signal: watch::Receiver) { while !*stop_signal.borrow() { - let restart_at = tokio::time::sleep(STATUS_EXCHANGE_INTERVAL); + let restart_at = Instant::now() + STATUS_EXCHANGE_INTERVAL; + // Update local node status that is exchanged. self.update_local_status(); - let local_status: NodeStatus = self.local_status.load().as_ref().clone(); + + let local_status: NodeStatus = self.local_status.read().unwrap().clone(); let _ = self - .rpc + .rpc_helper() .broadcast( &self.system_endpoint, SystemRpc::AdvertiseStatus(local_status), - RequestStrategy::with_priority(PRIO_HIGH), + RequestStrategy::with_priority(PRIO_HIGH) + .with_custom_timeout(STATUS_EXCHANGE_INTERVAL), ) .await; select! { - _ = restart_at.fuse() => {}, - _ = stop_signal.changed().fuse() => {}, + _ = tokio::time::sleep_until(restart_at.into()) => {}, + _ = stop_signal.changed() => {}, } } } async fn discovery_loop(self: &Arc, mut stop_signal: watch::Receiver) { while !*stop_signal.borrow() { - let not_configured = self.ring.borrow().layout.check().is_err(); - let no_peers = self.fullmesh.get_peer_list().len() < self.replication_factor; - let expected_n_nodes = self.ring.borrow().layout.num_nodes(); - let bad_peers = self - .fullmesh + let n_connected = self + .peering .get_peer_list() .iter() .filter(|p| p.is_up()) - .count() != expected_n_nodes; + .count(); + + let not_configured = !self.cluster_layout().is_check_ok(); + let no_peers = n_connected < self.replication_factor.into(); + let expected_n_nodes = self.cluster_layout().all_nodes().len(); + let bad_peers = n_connected != expected_n_nodes; if not_configured || no_peers || bad_peers { info!("Doing a bootstrap/discovery step (not_configured: {}, no_peers: {}, bad_peers: {})", not_configured, no_peers, bad_peers); @@ -779,6 +683,14 @@ impl System { } } + if !not_configured && !no_peers { + // If the layout is configured, and we already have some connections + // to other nodes in the cluster, we can skip trying to connect to + // nodes that are not in the cluster layout. + let layout = self.cluster_layout(); + ping_list.retain(|(id, _)| layout.all_nodes().contains(&(*id).into())); + } + for (node_id, node_addr) in ping_list { let self2 = self.clone(); tokio::spawn(async move { @@ -799,10 +711,9 @@ impl System { #[cfg(feature = "kubernetes-discovery")] tokio::spawn(self.clone().advertise_to_kubernetes()); - let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL); select! { - _ = restart_at.fuse() => {}, - _ = stop_signal.changed().fuse() => {}, + _ = tokio::time::sleep(DISCOVERY_INTERVAL) => {}, + _ = stop_signal.changed() => {}, } } } @@ -811,10 +722,13 @@ impl System { // Prepare new peer list to save to file // It is a vec of tuples (node ID as Uuid, node SocketAddr) let mut peer_list = self - .fullmesh + .peering .get_peer_list() .iter() - .map(|n| (n.id.into(), n.addr)) + .filter_map(|n| match n.state { + PeerConnState::Connected { addr } => Some((n.id.into(), addr)), + _ => None, + }) .collect::>(); // Before doing it, we read the current peer list file (if it exists) @@ -832,48 +746,48 @@ impl System { .save_async(&PeerList(peer_list)) .await } - - async fn pull_cluster_layout(self: Arc, peer: Uuid) { - let resp = self - .rpc - .call( - &self.system_endpoint, - peer, - SystemRpc::PullClusterLayout, - RequestStrategy::with_priority(PRIO_HIGH), - ) - .await; - if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp { - let _: Result<_, _> = self.handle_advertise_cluster_layout(&layout).await; - } - } } -#[async_trait] impl EndpointHandler for System { async fn handle(self: &Arc, msg: &SystemRpc, from: NodeID) -> Result { match msg { + // ---- system functions -> System ---- SystemRpc::Connect(node) => self.handle_connect(node).await, - SystemRpc::PullClusterLayout => Ok(self.handle_pull_cluster_layout()), SystemRpc::AdvertiseStatus(adv) => self.handle_advertise_status(from.into(), adv).await, - SystemRpc::AdvertiseClusterLayout(adv) => { - self.clone().handle_advertise_cluster_layout(adv).await - } SystemRpc::GetKnownNodes => Ok(self.handle_get_known_nodes()), + + // ---- layout functions -> LayoutManager ---- + SystemRpc::PullClusterLayout => Ok(self.layout_manager.handle_pull_cluster_layout()), + SystemRpc::AdvertiseClusterLayout(adv) => { + self.layout_manager + .handle_advertise_cluster_layout(adv) + .await + } + SystemRpc::PullClusterLayoutTrackers => { + Ok(self.layout_manager.handle_pull_cluster_layout_trackers()) + } + SystemRpc::AdvertiseClusterLayoutTrackers(adv) => { + self.layout_manager + .handle_advertise_cluster_layout_trackers(adv) + .await + } + + // ---- other -> Error ---- m => Err(Error::unexpected_rpc_message(m)), } } } impl NodeStatus { - fn initial(replication_factor: usize, layout: &ClusterLayout) -> Self { + fn initial(replication_factor: ReplicationFactor, layout_manager: &LayoutManager) -> Self { NodeStatus { - hostname: gethostname::gethostname() - .into_string() - .unwrap_or_else(|_| "".to_string()), - replication_factor, - cluster_layout_version: layout.version, - cluster_layout_staging_hash: layout.staging_hash, + hostname: Some( + gethostname::gethostname() + .into_string() + .unwrap_or_else(|_| "".to_string()), + ), + replication_factor: replication_factor.into(), + layout_digest: layout_manager.layout().digest(), meta_disk_avail: None, data_disk_avail: None, } @@ -881,32 +795,37 @@ impl NodeStatus { fn unknown() -> Self { NodeStatus { - hostname: "?".to_string(), + hostname: None, replication_factor: 0, - cluster_layout_version: 0, - cluster_layout_staging_hash: Hash::from([0u8; 32]), + layout_digest: Default::default(), meta_disk_avail: None, data_disk_avail: None, } } - fn update_disk_usage( - &mut self, - meta_dir: &Path, - data_dir: &DataDirEnum, - metrics: &SystemMetrics, - ) { + fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &DataDirEnum) { use nix::sys::statvfs::statvfs; + + // The HashMap used below requires a filesystem identifier from statfs (instead of statvfs) on FreeBSD, as + // FreeBSD's statvfs filesystem identifier is "not meaningful in this implementation" (man 3 statvfs). + + #[cfg(target_os = "freebsd")] + let get_filesystem_id = |path: &Path| match nix::sys::statfs::statfs(path) { + Ok(fs) => Some(fs.filesystem_id()), + Err(_) => None, + }; + let mount_avail = |path: &Path| match statvfs(path) { Ok(x) => { - let avail = x.blocks_available() * x.fragment_size() as u64; - let total = x.blocks() * x.fragment_size() as u64; + let avail = x.blocks_available() as u64 * x.fragment_size() as u64; + let total = x.blocks() as u64 * x.fragment_size() as u64; Some((x.filesystem_id(), avail, total)) } Err(_) => None, }; self.meta_disk_avail = mount_avail(meta_dir).map(|(_, a, t)| (a, t)); + self.data_disk_avail = match data_dir { DataDirEnum::Single(dir) => mount_avail(dir).map(|(_, a, t)| (a, t)), DataDirEnum::Multiple(dirs) => (|| { @@ -917,12 +836,25 @@ impl NodeStatus { if dir.capacity.is_none() { continue; } + + #[cfg(not(target_os = "freebsd"))] match mount_avail(&dir.path) { Some((fsid, avail, total)) => { mounts.insert(fsid, (avail, total)); } None => return None, } + + #[cfg(target_os = "freebsd")] + match get_filesystem_id(&dir.path) { + Some(fsid) => match mount_avail(&dir.path) { + Some((_, avail, total)) => { + mounts.insert(fsid, (avail, total)); + } + None => return None, + }, + None => return None, + } } Some( mounts @@ -931,36 +863,78 @@ impl NodeStatus { ) })(), }; - - if let Some((avail, total)) = self.meta_disk_avail { - metrics - .values - .meta_disk_avail - .store(avail, Ordering::Relaxed); - metrics - .values - .meta_disk_total - .store(total, Ordering::Relaxed); - } - if let Some((avail, total)) = self.data_disk_avail { - metrics - .values - .data_disk_avail - .store(avail, Ordering::Relaxed); - metrics - .values - .data_disk_total - .store(total, Ordering::Relaxed); - } } } -fn get_default_ip() -> Option { +/// Obtain the list of currently available IP addresses on all non-loopback +/// interfaces, optionally filtering them to be inside a given IpNet. +fn get_default_ip(filter_ipnet: Option) -> Option { pnet_datalink::interfaces() - .iter() - .find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) - .and_then(|e| e.ips.first()) - .map(|a| a.ip()) + .into_iter() + // filter down and loopback interfaces + .filter(|i| i.is_up() && !i.is_loopback()) + // get all IPs + .flat_map(|e| e.ips) + // optionally, filter to be inside filter_ipnet + .find(|ipn| { + filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none() + }) + .map(|ipn| ipn.ip()) +} + +fn get_rpc_public_addr(config: &Config) -> Option { + match &config.rpc_public_addr { + Some(a_str) => { + use std::net::ToSocketAddrs; + match a_str.to_socket_addrs() { + Err(e) => { + error!( + "Cannot resolve rpc_public_addr {} from config file: {}.", + a_str, e + ); + None + } + Ok(a) => { + let a = a.collect::>(); + if a.is_empty() { + error!("rpc_public_addr {} resolve to no known IP address", a_str); + } + if a.len() > 1 { + warn!("Multiple possible resolutions for rpc_public_addr: {:?}. Taking the first one.", a); + } + a.into_iter().next() + } + } + } + None => { + // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`. + let filter_subnet: Option = config + .rpc_public_addr_subnet + .as_ref() + .and_then(|filter_subnet_str| match filter_subnet_str.parse::() { + Ok(filter_subnet) => { + let filter_subnet_trunc = filter_subnet.trunc(); + if filter_subnet_trunc != filter_subnet { + warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc()); + } + Some(filter_subnet_trunc) + } + Err(e) => { + panic!( + "Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.", + filter_subnet_str, e + ); + } + }); + + let addr = get_default_ip(filter_subnet) + .map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); + if let Some(a) = addr { + warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); + } + addr + } + } } async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> { @@ -985,7 +959,7 @@ async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> { fn connect_error_message( addr: SocketAddr, pubkey: ed25519::PublicKey, - e: netapp::error::Error, + e: garage_net::error::Error, ) -> String { format!("Error establishing RPC connection to remote node: {}@{}.\nThis can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret.\n{}", hex::encode(pubkey), addr, e) } diff --git a/src/rpc/system_metrics.rs b/src/rpc/system_metrics.rs index af81b71f..a64daec8 100644 --- a/src/rpc/system_metrics.rs +++ b/src/rpc/system_metrics.rs @@ -1,32 +1,57 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; use opentelemetry::{global, metrics::*, KeyValue}; +use crate::system::{ClusterHealthStatus, System}; + /// TableMetrics reference all counter used for metrics pub struct SystemMetrics { + // Static values pub(crate) _garage_build_info: ValueObserver, pub(crate) _replication_factor: ValueObserver, + + // Disk space values from System::local_status pub(crate) _disk_avail: ValueObserver, pub(crate) _disk_total: ValueObserver, - pub(crate) values: Arc, -} -#[derive(Default)] -pub struct SystemMetricsValues { - pub(crate) data_disk_total: AtomicU64, - pub(crate) data_disk_avail: AtomicU64, - pub(crate) meta_disk_total: AtomicU64, - pub(crate) meta_disk_avail: AtomicU64, + // Health report from System::health() + pub(crate) _cluster_healthy: ValueObserver, + pub(crate) _cluster_available: ValueObserver, + pub(crate) _known_nodes: ValueObserver, + pub(crate) _connected_nodes: ValueObserver, + pub(crate) _storage_nodes: ValueObserver, + pub(crate) _storage_nodes_ok: ValueObserver, + pub(crate) _partitions: ValueObserver, + pub(crate) _partitions_quorum: ValueObserver, + pub(crate) _partitions_all_ok: ValueObserver, + + // Status report for individual cluster nodes + pub(crate) _layout_node_connected: ValueObserver, + pub(crate) _layout_node_disconnected_time: ValueObserver, } impl SystemMetrics { - pub fn new(replication_factor: usize) -> Self { + pub fn new(system: Arc) -> Self { let meter = global::meter("garage_system"); - let values = Arc::new(SystemMetricsValues::default()); - let values1 = values.clone(); - let values2 = values.clone(); + + let health_cache = RwLock::new((Instant::now(), system.health())); + let system2 = system.clone(); + let get_health = Arc::new(move || { + { + let cache = health_cache.read().unwrap(); + if cache.0 > Instant::now() - Duration::from_secs(1) { + return cache.1; + } + } + + let health = system2.health(); + *health_cache.write().unwrap() = (Instant::now(), health); + health + }); + Self { + // Static values _garage_build_info: meter .u64_value_observer("garage_build_info", move |observer| { observer.observe( @@ -39,39 +64,239 @@ impl SystemMetrics { }) .with_description("Garage build info") .init(), - _replication_factor: meter - .u64_value_observer("garage_replication_factor", move |observer| { - observer.observe(replication_factor as u64, &[]) + _replication_factor: { + let replication_factor = system.replication_factor; + meter + .u64_value_observer("garage_replication_factor", move |observer| { + observer.observe(replication_factor.replication_factor() as u64, &[]) + }) + .with_description("Garage replication factor setting") + .init() + }, + + // Disk space values from System::local_status + _disk_avail: { + let system = system.clone(); + meter + .u64_value_observer("garage_local_disk_avail", move |observer| { + let st = system.local_status.read().unwrap(); + if let Some((avail, _total)) = st.data_disk_avail { + observer.observe(avail, &[KeyValue::new("volume", "data")]); + } + if let Some((avail, _total)) = st.meta_disk_avail { + observer.observe(avail, &[KeyValue::new("volume", "metadata")]); + } + }) + .with_description("Garage available disk space on each node") + .init() + }, + _disk_total: { + let system = system.clone(); + meter + .u64_value_observer("garage_local_disk_total", move |observer| { + let st = system.local_status.read().unwrap(); + if let Some((_avail, total)) = st.data_disk_avail { + observer.observe(total, &[KeyValue::new("volume", "data")]); + } + if let Some((_avail, total)) = st.meta_disk_avail { + observer.observe(total, &[KeyValue::new("volume", "metadata")]); + } + }) + .with_description("Garage total disk space on each node") + .init() + }, + + // Health report from System::() + _cluster_healthy: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_healthy", move |observer| { + let h = get_health(); + if h.status == ClusterHealthStatus::Healthy { + observer.observe(1, &[]); + } else { + observer.observe(0, &[]); + } + }) + .with_description("Whether all storage nodes are connected") + .init() + }, + _cluster_available: { + let get_health = get_health.clone(); + meter.u64_value_observer("cluster_available", move |observer| { + let h = get_health(); + if h.status != ClusterHealthStatus::Unavailable { + observer.observe(1, &[]); + } else { + observer.observe(0, &[]); + } }) - .with_description("Garage replication factor setting") - .init(), - _disk_avail: meter - .u64_value_observer("garage_local_disk_avail", move |observer| { - match values1.data_disk_avail.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "data")]), - }; - match values1.meta_disk_avail.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), - }; - }) - .with_description("Garage available disk space on each node") - .init(), - _disk_total: meter - .u64_value_observer("garage_local_disk_total", move |observer| { - match values2.data_disk_total.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "data")]), - }; - match values2.meta_disk_total.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), - }; - }) - .with_description("Garage total disk space on each node") - .init(), - values, + .with_description("Whether all requests can be served, even if some storage nodes are disconnected") + .init() + }, + _known_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_known_nodes", move |observer| { + let h = get_health(); + observer.observe(h.known_nodes as u64, &[]); + }) + .with_description("Number of nodes already seen once in the cluster") + .init() + }, + _connected_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_connected_nodes", move |observer| { + let h = get_health(); + observer.observe(h.connected_nodes as u64, &[]); + }) + .with_description("Number of nodes currently connected") + .init() + }, + _storage_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_storage_nodes", move |observer| { + let h = get_health(); + observer.observe(h.storage_nodes as u64, &[]); + }) + .with_description("Number of storage nodes declared in the current layout") + .init() + }, + _storage_nodes_ok: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_storage_nodes_ok", move |observer| { + let h = get_health(); + observer.observe(h.storage_nodes_ok as u64, &[]); + }) + .with_description("Number of storage nodes currently connected") + .init() + }, + _partitions: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions", move |observer| { + let h = get_health(); + observer.observe(h.partitions as u64, &[]); + }) + .with_description("Number of partitions in the layout") + .init() + }, + _partitions_quorum: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions_quorum", move |observer| { + let h = get_health(); + observer.observe(h.partitions_quorum as u64, &[]); + }) + .with_description( + "Number of partitions for which we have a quorum of connected nodes", + ) + .init() + }, + _partitions_all_ok: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions_all_ok", move |observer| { + let h = get_health(); + observer.observe(h.partitions_all_ok as u64, &[]); + }) + .with_description( + "Number of partitions for which all storage nodes are connected", + ) + .init() + }, + + // Status report for individual cluster nodes + _layout_node_connected: { + let system = system.clone(); + meter + .u64_value_observer("cluster_layout_node_connected", move |observer| { + let layout = system.cluster_layout(); + let nodes = system.get_known_nodes(); + for id in layout.all_nodes().iter() { + let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; + if let Some(role) = + layout.current().roles.get(id).and_then(|r| r.0.as_ref()) + { + kv.push(KeyValue::new("role_zone", role.zone.clone())); + match role.capacity { + Some(cap) => { + kv.push(KeyValue::new("role_capacity", cap as i64)); + kv.push(KeyValue::new("role_gateway", 0)); + } + None => { + kv.push(KeyValue::new("role_gateway", 1)); + } + } + } + + let value; + if let Some(node) = nodes.iter().find(|n| n.id == *id) { + // TODO: if we add address and hostname, and those change, we + // get duplicate metrics, due to bad otel aggregation :( + // Can probably be fixed when we upgrade opentelemetry + // kv.push(KeyValue::new("address", node.addr.to_string())); + // kv.push(KeyValue::new( + // "hostname", + // node.status.hostname.clone(), + // )); + value = if node.is_up { 1 } else { 0 }; + } else { + value = 0; + } + + observer.observe(value, &kv); + } + }) + .with_description("Connection status for nodes in the cluster layout") + .init() + }, + _layout_node_disconnected_time: { + let system = system.clone(); + meter + .u64_value_observer("cluster_layout_node_disconnected_time", move |observer| { + let layout = system.cluster_layout(); + let nodes = system.get_known_nodes(); + for id in layout.all_nodes().iter() { + let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; + if let Some(role) = + layout.current().roles.get(id).and_then(|r| r.0.as_ref()) + { + kv.push(KeyValue::new("role_zone", role.zone.clone())); + match role.capacity { + Some(cap) => { + kv.push(KeyValue::new("role_capacity", cap as i64)); + kv.push(KeyValue::new("role_gateway", 0)); + } + None => { + kv.push(KeyValue::new("role_gateway", 1)); + } + } + } + + if let Some(node) = nodes.iter().find(|n| n.id == *id) { + // TODO: see comment above + // kv.push(KeyValue::new("address", node.addr.to_string())); + // kv.push(KeyValue::new( + // "hostname", + // node.status.hostname.clone(), + // )); + if node.is_up { + observer.observe(0, &kv); + } else if let Some(secs) = node.last_seen_secs_ago { + observer.observe(secs, &kv); + } + } + } + }) + .with_description( + "Time (in seconds) since last connection to nodes in the cluster layout", + ) + .init() + }, } } } diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 0d2a3103..478dbd18 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -18,19 +18,18 @@ garage_db.workspace = true garage_rpc.workspace = true garage_util.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true -async-trait = "0.1.7" -arc-swap = "1.0" -bytes = "1.0" -hex = "0.4" -hexdump = "0.1" -tracing = "0.1" -rand = "0.8" +async-trait.workspace = true +arc-swap.workspace = true +hex.workspace = true +hexdump.workspace = true +tracing.workspace = true +rand.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true +serde_bytes.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true diff --git a/src/table/data.rs b/src/table/data.rs index bbfdf58b..1d0308ce 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -6,7 +6,6 @@ use serde_bytes::ByteBuf; use tokio::sync::Notify; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; use garage_util::data::*; use garage_util::error::*; @@ -36,7 +35,7 @@ pub struct TableData { pub(crate) insert_queue: db::Tree, pub(crate) insert_queue_notify: Arc, - pub(crate) gc_todo: CountedTree, + pub(crate) gc_todo: db::Tree, pub(crate) metrics: TableMetrics, } @@ -61,7 +60,6 @@ impl TableData { let gc_todo = db .open_tree(format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open GC DB tree"); - let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2"); let metrics = TableMetrics::new( F::TABLE_NAME, @@ -254,7 +252,8 @@ impl TableData { // of the GC algorithm, as in all cases GC is suspended if // any node of the partition is unavailable. let pk_hash = Hash::try_from(&tree_key[..32]).unwrap(); - let nodes = self.replication.write_nodes(&pk_hash); + // TODO: this probably breaks when the layout changes + let nodes = self.replication.storage_nodes(&pk_hash); if nodes.first() == Some(&self.system.id) { GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?; } @@ -368,7 +367,7 @@ impl TableData { } } - pub fn gc_todo_len(&self) -> Result { - Ok(self.gc_todo.len()) + pub fn gc_todo_approximate_len(&self) -> Result { + Ok(self.gc_todo.approximate_len()?) } } diff --git a/src/table/gc.rs b/src/table/gc.rs index 5b9124a7..1f30bd76 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -4,13 +4,14 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; + use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use futures::future::join_all; use tokio::sync::watch; -use garage_db::counted_tree_hack::CountedTree; +use garage_db as db; use garage_util::background::*; use garage_util::data::*; @@ -152,7 +153,7 @@ impl TableGc { let mut partitions = HashMap::new(); for entry in entries { let pkh = Hash::try_from(&entry.key[..32]).unwrap(); - let mut nodes = self.data.replication.write_nodes(&pkh); + let mut nodes = self.data.replication.storage_nodes(&pkh); nodes.retain(|x| *x != self.system.id); nodes.sort(); @@ -227,10 +228,10 @@ impl TableGc { // GC'ing is not a critical function of the system, so it's not a big // deal if we can't do it right now. self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &nodes[..], + &nodes, GcRpc::Update(updates), RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()), ) @@ -248,31 +249,30 @@ impl TableGc { // it means that the garbage collection wasn't completed and has // to be retried later. self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &nodes[..], + &nodes, GcRpc::DeleteIfEqualHash(deletes), RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()), ) .await .err_context("GC: remote delete tombstones")?; - // GC has been successfull for all of these entries. + // GC has been successful for all of these entries. // We now remove them all from our local table and from the GC todo list. for item in items { self.data .delete_if_equal_hash(&item.key[..], item.value_hash) .err_context("GC: local delete tombstones")?; item.remove_if_equal(&self.data.gc_todo) - .err_context("GC: remove from todo list after successfull GC")?; + .err_context("GC: remove from todo list after successful GC")?; } Ok(()) } } -#[async_trait] impl EndpointHandler for TableGc { async fn handle(self: &Arc, message: &GcRpc, _from: NodeID) -> Result { match message { @@ -313,7 +313,7 @@ impl Worker for GcWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64), + queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64), ..Default::default() } } @@ -334,9 +334,9 @@ impl Worker for GcWorker { } } -/// An entry stored in the gc_todo Sled tree associated with the table +/// An entry stored in the gc_todo db tree associated with the table /// Contains helper function for parsing, saving, and removing -/// such entry in Sled +/// such entry in the db /// /// Format of an entry: /// - key = 8 bytes: timestamp of tombstone @@ -353,7 +353,7 @@ pub(crate) struct GcTodoEntry { } impl GcTodoEntry { - /// Creates a new GcTodoEntry (not saved in Sled) from its components: + /// Creates a new GcTodoEntry (not saved in the db) from its components: /// the key of an entry in the table, and the hash of the associated /// serialized value pub(crate) fn new(key: Vec, value_hash: Hash) -> Self { @@ -376,22 +376,24 @@ impl GcTodoEntry { } /// Saves the GcTodoEntry in the gc_todo tree - pub(crate) fn save(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { + pub(crate) fn save(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { gc_todo_tree.insert(self.todo_table_key(), self.value_hash.as_slice())?; Ok(()) } /// Removes the GcTodoEntry from the gc_todo tree if the /// hash of the serialized value is the same here as in the tree. - /// This is usefull to remove a todo entry only under the condition + /// This is useful to remove a todo entry only under the condition /// that it has not changed since the time it was read, i.e. /// what we have to do is still the same - pub(crate) fn remove_if_equal(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { - gc_todo_tree.compare_and_swap::<_, _, &[u8]>( - &self.todo_table_key(), - Some(self.value_hash), - None, - )?; + pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { + gc_todo_tree.db().transaction(|txn| { + let key = self.todo_table_key(); + if txn.get(gc_todo_tree, &key)?.as_deref() == Some(self.value_hash.as_slice()) { + txn.remove(gc_todo_tree, &key)?; + } + Ok(()) + })?; Ok(()) } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 4577f872..7ba1f007 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -13,7 +13,7 @@ use garage_util::data::*; use garage_util::encode::{nonversioned_decode, nonversioned_encode}; use garage_util::error::Error; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use crate::data::*; use crate::replication::*; @@ -31,14 +31,14 @@ pub struct MerkleUpdater { // - value = the hash of the full serialized item, if present, // or an empty vec if item is absent (deleted) // Fields in data: - // pub(crate) merkle_todo: sled::Tree, + // pub(crate) merkle_todo: db::Tree, // pub(crate) merkle_todo_notify: Notify, // Content of the merkle tree: items where // - key = .bytes() for MerkleNodeKey // - value = serialization of a MerkleNode, assumed to be MerkleNode::empty if not found // Field in data: - // pub(crate) merkle_tree: sled::Tree, + // pub(crate) merkle_tree: db::Tree, empty_node_hash: Hash, } @@ -287,16 +287,12 @@ impl MerkleUpdater { MerkleNode::decode_opt(&ent) } - pub fn merkle_tree_len(&self) -> Result { - Ok(self.data.merkle_tree.len()?) + pub fn merkle_tree_approximate_len(&self) -> Result { + Ok(self.data.merkle_tree.approximate_len()?) } - pub fn merkle_tree_fast_len(&self) -> Result, Error> { - Ok(self.data.merkle_tree.fast_len()?) - } - - pub fn todo_len(&self) -> Result { - Ok(self.data.merkle_todo.len()?) + pub fn todo_approximate_len(&self) -> Result { + Ok(self.data.merkle_todo.approximate_len()?) } } @@ -310,7 +306,7 @@ impl Worker for MerkleWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.0.todo_len().unwrap_or(0) as u64), + queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64), ..Default::default() } } diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 8318a84f..78593202 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,7 +1,6 @@ use opentelemetry::{global, metrics::*, KeyValue}; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct TableMetrics { @@ -27,7 +26,7 @@ impl TableMetrics { store: db::Tree, merkle_tree: db::Tree, merkle_todo: db::Tree, - gc_todo: CountedTree, + gc_todo: db::Tree, ) -> Self { let meter = global::meter(table_name); TableMetrics { @@ -35,9 +34,9 @@ impl TableMetrics { .u64_value_observer( "table.size", move |observer| { - if let Ok(Some(v)) = store.fast_len() { + if let Ok(value) = store.approximate_len() { observer.observe( - v as u64, + value as u64, &[KeyValue::new("table_name", table_name)], ); } @@ -49,9 +48,9 @@ impl TableMetrics { .u64_value_observer( "table.merkle_tree_size", move |observer| { - if let Ok(Some(v)) = merkle_tree.fast_len() { + if let Ok(value) = merkle_tree.approximate_len() { observer.observe( - v as u64, + value as u64, &[KeyValue::new("table_name", table_name)], ); } @@ -63,7 +62,7 @@ impl TableMetrics { .u64_value_observer( "table.merkle_updater_todo_queue_length", move |observer| { - if let Ok(v) = merkle_todo.len() { + if let Ok(v) = merkle_todo.approximate_len() { observer.observe( v as u64, &[KeyValue::new("table_name", table_name)], @@ -77,10 +76,12 @@ impl TableMetrics { .u64_value_observer( "table.gc_todo_queue_length", move |observer| { - observer.observe( - gc_todo.len() as u64, - &[KeyValue::new("table_name", table_name)], - ); + if let Ok(value) = gc_todo.approximate_len() { + observer.observe( + value as u64, + &[KeyValue::new("table_name", table_name)], + ); + } }, ) .with_description("Table garbage collector TODO queue length") diff --git a/src/table/queue.rs b/src/table/queue.rs index ffe0a4a7..7ef1f16e 100644 --- a/src/table/queue.rs +++ b/src/table/queue.rs @@ -27,7 +27,7 @@ impl Worker for InsertQueueWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64), + queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64), ..Default::default() } } diff --git a/src/table/replication/fullcopy.rs b/src/table/replication/fullcopy.rs index 18682ace..1e52bb47 100644 --- a/src/table/replication/fullcopy.rs +++ b/src/table/replication/fullcopy.rs @@ -1,24 +1,36 @@ use std::sync::Arc; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_rpc::system::System; use garage_util::data::*; use crate::replication::*; +// TODO: find a way to track layout changes for this as well +// The hard thing is that this data is stored also on gateway nodes, +// whereas sharded data is stored only on non-Gateway nodes (storage nodes) +// Also we want to be more tolerant to failures of gateways so we don't +// want to do too much holding back of data when progress of gateway +// nodes is not reported in the layout history's ack/sync/sync_ack maps. + /// Full replication schema: all nodes store everything -/// Writes are disseminated in an epidemic manner in the network /// Advantage: do all reads locally, extremely fast /// Inconvenient: only suitable to reasonably small tables +/// Inconvenient: if some writes fail, nodes will read outdated data #[derive(Clone)] pub struct TableFullReplication { /// The membership manager of this node pub system: Arc, - /// Max number of faults allowed while replicating a record - pub max_faults: usize, } impl TableReplication for TableFullReplication { + type WriteSets = Vec>; + + fn storage_nodes(&self, _hash: &Hash) -> Vec { + let layout = self.system.cluster_layout(); + layout.current().all_nodes().to_vec() + } + fn read_nodes(&self, _hash: &Hash) -> Vec { vec![self.system.id] } @@ -26,26 +38,36 @@ impl TableReplication for TableFullReplication { 1 } - fn write_nodes(&self, _hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.layout.node_ids().to_vec() + fn write_sets(&self, hash: &Hash) -> Self::WriteSets { + vec![self.storage_nodes(hash)] } fn write_quorum(&self) -> usize { - let nmembers = self.system.ring.borrow().layout.node_ids().len(); - if nmembers > self.max_faults { - nmembers - self.max_faults + let nmembers = self.system.cluster_layout().current().all_nodes().len(); + + let max_faults = if nmembers > 1 { 1 } else { 0 }; + + if nmembers > max_faults { + nmembers - max_faults } else { 1 } } - fn max_write_errors(&self) -> usize { - self.max_faults - } fn partition_of(&self, _hash: &Hash) -> Partition { 0u16 } - fn partitions(&self) -> Vec<(Partition, Hash)> { - vec![(0u16, [0u8; 32].into())] + + fn sync_partitions(&self) -> SyncPartitions { + let layout = self.system.cluster_layout(); + let layout_version = layout.current().version; + SyncPartitions { + layout_version, + partitions: vec![SyncPartition { + partition: 0u16, + first_hash: [0u8; 32].into(), + last_hash: [0xff; 32].into(), + storage_sets: vec![layout.current().all_nodes().to_vec()], + }], + } } } diff --git a/src/table/replication/parameters.rs b/src/table/replication/parameters.rs index f00815a2..3649fad3 100644 --- a/src/table/replication/parameters.rs +++ b/src/table/replication/parameters.rs @@ -1,25 +1,43 @@ -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_util::data::*; /// Trait to describe how a table shall be replicated pub trait TableReplication: Send + Sync + 'static { + type WriteSets: AsRef>> + AsMut>> + Send + Sync + 'static; + // See examples in table_sharded.rs and table_fullcopy.rs // To understand various replication methods + /// The entire list of all nodes that store a partition + fn storage_nodes(&self, hash: &Hash) -> Vec; + /// Which nodes to send read requests to fn read_nodes(&self, hash: &Hash) -> Vec; - /// Responses needed to consider a read succesfull + /// Responses needed to consider a read successful fn read_quorum(&self) -> usize; /// Which nodes to send writes to - fn write_nodes(&self, hash: &Hash) -> Vec; - /// Responses needed to consider a write succesfull + fn write_sets(&self, hash: &Hash) -> Self::WriteSets; + /// Responses needed to consider a write successful in each set fn write_quorum(&self) -> usize; - fn max_write_errors(&self) -> usize; // Accessing partitions, for Merkle tree & sync /// Get partition for data with given hash fn partition_of(&self, hash: &Hash) -> Partition; - /// List of existing partitions - fn partitions(&self) -> Vec<(Partition, Hash)>; + /// List of partitions and nodes to sync with in current layout + fn sync_partitions(&self) -> SyncPartitions; +} + +#[derive(Debug)] +pub struct SyncPartitions { + pub layout_version: u64, + pub partitions: Vec, +} + +#[derive(Debug)] +pub struct SyncPartition { + pub partition: Partition, + pub first_hash: Hash, + pub last_hash: Hash, + pub storage_sets: Vec>, } diff --git a/src/table/replication/sharded.rs b/src/table/replication/sharded.rs index 1cf964af..e0245949 100644 --- a/src/table/replication/sharded.rs +++ b/src/table/replication/sharded.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_rpc::system::System; use garage_util::data::*; @@ -25,29 +25,59 @@ pub struct TableShardedReplication { } impl TableReplication for TableShardedReplication { + type WriteSets = WriteLock>>; + + fn storage_nodes(&self, hash: &Hash) -> Vec { + self.system.cluster_layout().storage_nodes_of(hash) + } + fn read_nodes(&self, hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.get_nodes(hash, self.replication_factor) + self.system.cluster_layout().read_nodes_of(hash) } fn read_quorum(&self) -> usize { self.read_quorum } - fn write_nodes(&self, hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.get_nodes(hash, self.replication_factor) + fn write_sets(&self, hash: &Hash) -> Self::WriteSets { + self.system.layout_manager.write_sets_of(hash) } fn write_quorum(&self) -> usize { self.write_quorum } - fn max_write_errors(&self) -> usize { - self.replication_factor - self.write_quorum - } fn partition_of(&self, hash: &Hash) -> Partition { - self.system.ring.borrow().partition_of(hash) + self.system.cluster_layout().current().partition_of(hash) } - fn partitions(&self) -> Vec<(Partition, Hash)> { - self.system.ring.borrow().partitions() + + fn sync_partitions(&self) -> SyncPartitions { + let layout = self.system.cluster_layout(); + let layout_version = layout.ack_map_min(); + + let mut partitions = layout + .current() + .partitions() + .map(|(partition, first_hash)| { + let storage_sets = layout.storage_sets_of(&first_hash); + SyncPartition { + partition, + first_hash, + last_hash: [0u8; 32].into(), // filled in just after + storage_sets, + } + }) + .collect::>(); + + for i in 0..partitions.len() { + partitions[i].last_hash = if i + 1 < partitions.len() { + partitions[i + 1].first_hash + } else { + [0xFFu8; 32].into() + }; + } + + SyncPartitions { + layout_version, + partitions, + } } } diff --git a/src/table/sync.rs b/src/table/sync.rs index 92a353c6..2d43b9fc 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -6,18 +6,19 @@ use arc_swap::ArcSwapOption; use async_trait::async_trait; use futures_util::stream::*; use opentelemetry::KeyValue; -use rand::Rng; +use rand::prelude::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use tokio::select; -use tokio::sync::{mpsc, watch}; +use tokio::sync::{mpsc, watch, Notify}; use garage_util::background::*; use garage_util::data::*; use garage_util::encode::{debug_serialize, nonversioned_encode}; use garage_util::error::{Error, OkOrMessage}; -use garage_rpc::ring::*; +use garage_rpc::layout::*; +use garage_rpc::rpc_helper::QuorumSetResultTracker; use garage_rpc::system::System; use garage_rpc::*; @@ -52,16 +53,6 @@ impl Rpc for SyncRpc { type Response = Result; } -#[derive(Debug, Clone)] -struct TodoPartition { - partition: Partition, - begin: Hash, - end: Hash, - - // Are we a node that stores this partition or not? - retain: bool, -} - impl TableSyncer { pub(crate) fn new( system: Arc, @@ -91,10 +82,10 @@ impl TableSyncer { bg.spawn_worker(SyncWorker { syncer: self.clone(), - ring_recv: self.system.ring.clone(), - ring: self.system.ring.borrow().clone(), + layout_notify: self.system.layout_notify(), + layout_digest: self.system.cluster_layout().sync_digest(), add_full_sync_rx, - todo: vec![], + todo: None, next_full_sync: Instant::now() + Duration::from_secs(20), }); } @@ -112,53 +103,56 @@ impl TableSyncer { async fn sync_partition( self: &Arc, - partition: &TodoPartition, + partition: &SyncPartition, must_exit: &mut watch::Receiver, ) -> Result<(), Error> { - if partition.retain { - let my_id = self.system.id; - - let nodes = self - .data - .replication - .write_nodes(&partition.begin) - .into_iter() - .filter(|node| *node != my_id) - .collect::>(); + let my_id = self.system.id; + let retain = partition.storage_sets.iter().any(|x| x.contains(&my_id)); + if retain { debug!( "({}) Syncing {:?} with {:?}...", F::TABLE_NAME, partition, - nodes + partition.storage_sets ); - let mut sync_futures = nodes - .iter() + let mut result_tracker = QuorumSetResultTracker::new( + &partition.storage_sets, + self.data.replication.write_quorum(), + ); + + let mut sync_futures = result_tracker + .nodes + .keys() + .copied() .map(|node| { - self.clone() - .do_sync_with(partition.clone(), *node, must_exit.clone()) + let must_exit = must_exit.clone(); + async move { + if node == my_id { + (node, Ok(())) + } else { + (node, self.do_sync_with(partition, node, must_exit).await) + } + } }) .collect::>(); - let mut n_errors = 0; - while let Some(r) = sync_futures.next().await { - if let Err(e) = r { - n_errors += 1; - warn!("({}) Sync error: {}", F::TABLE_NAME, e); + while let Some((node, res)) = sync_futures.next().await { + if let Err(e) = &res { + warn!("({}) Sync error with {:?}: {}", F::TABLE_NAME, node, e); } + result_tracker.register_result(node, res); } - if n_errors > self.data.replication.max_write_errors() { - return Err(Error::Message(format!( - "Sync failed with too many nodes (should have been: {:?}).", - nodes - ))); + + if result_tracker.too_many_failures() { + Err(result_tracker.quorum_error()) + } else { + Ok(()) } } else { - self.offload_partition(&partition.begin, &partition.end, must_exit) - .await?; + self.offload_partition(&partition.first_hash, &partition.last_hash, must_exit) + .await } - - Ok(()) } // Offload partition: this partition is not something we are storing, @@ -188,12 +182,7 @@ impl TableSyncer { } if !items.is_empty() { - let nodes = self - .data - .replication - .write_nodes(begin) - .into_iter() - .collect::>(); + let nodes = self.data.replication.storage_nodes(begin); if nodes.contains(&self.system.id) { warn!( "({}) Interrupting offload as partitions seem to have changed", @@ -217,7 +206,7 @@ impl TableSyncer { end, counter ); - self.offload_items(&items, &nodes[..]).await?; + self.offload_items(&items, &nodes).await?; } else { break; } @@ -244,7 +233,7 @@ impl TableSyncer { } self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, nodes, @@ -284,8 +273,8 @@ impl TableSyncer { } async fn do_sync_with( - self: Arc, - partition: TodoPartition, + self: &Arc, + partition: &SyncPartition, who: Uuid, must_exit: watch::Receiver, ) -> Result<(), Error> { @@ -305,7 +294,7 @@ impl TableSyncer { // If so, do nothing. let root_resp = self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -327,7 +316,7 @@ impl TableSyncer { SyncRpc::RootCkDifferent(true) => VecDeque::from(vec![root_ck_key]), x => { return Err(Error::Message(format!( - "Invalid respone to RootCkHash RPC: {}", + "Invalid response to RootCkHash RPC: {}", debug_serialize(x) ))); } @@ -361,7 +350,7 @@ impl TableSyncer { // and compare it with local node let remote_node = match self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -373,7 +362,7 @@ impl TableSyncer { SyncRpc::Node(_, node) => node, x => { return Err(Error::Message(format!( - "Invalid respone to GetNode RPC: {}", + "Invalid response to GetNode RPC: {}", debug_serialize(x) ))); } @@ -437,7 +426,7 @@ impl TableSyncer { let rpc_resp = self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -455,7 +444,6 @@ impl TableSyncer { // ======= SYNCHRONIZATION PROCEDURE -- RECEIVER SIDE ====== -#[async_trait] impl EndpointHandler for TableSyncer { async fn handle(self: &Arc, message: &SyncRpc, from: NodeID) -> Result { match message { @@ -492,75 +480,41 @@ impl EndpointHandler for TableSync struct SyncWorker { syncer: Arc>, - ring_recv: watch::Receiver>, - ring: Arc, + + layout_notify: Arc, + layout_digest: SyncLayoutDigest, + add_full_sync_rx: mpsc::UnboundedReceiver<()>, - todo: Vec, next_full_sync: Instant, + + todo: Option, } impl SyncWorker { - fn add_full_sync(&mut self) { - let system = &self.syncer.system; - let data = &self.syncer.data; - - let my_id = system.id; - - self.todo.clear(); - - let partitions = data.replication.partitions(); - - for i in 0..partitions.len() { - let begin = partitions[i].1; - - let end = if i + 1 < partitions.len() { - partitions[i + 1].1 - } else { - [0xFFu8; 32].into() - }; - - let nodes = data.replication.write_nodes(&begin); - - let retain = nodes.contains(&my_id); - if !retain { - // Check if we have some data to send, otherwise skip - match data.store.range(begin..end) { - Ok(mut iter) => { - if iter.next().is_none() { - continue; - } - } - Err(e) => { - warn!("DB error in add_full_sync: {}", e); - continue; - } - } - } - - self.todo.push(TodoPartition { - partition: partitions[i].0, - begin, - end, - retain, - }); + fn check_add_full_sync(&mut self) { + let layout_digest = self.syncer.system.cluster_layout().sync_digest(); + if layout_digest != self.layout_digest { + self.layout_digest = layout_digest; + info!( + "({}) Layout versions changed ({:?}), adding full sync to syncer todo list", + F::TABLE_NAME, + layout_digest, + ); + self.add_full_sync(); } - - self.next_full_sync = Instant::now() + ANTI_ENTROPY_INTERVAL; } - fn pop_task(&mut self) -> Option { - if self.todo.is_empty() { - return None; - } + fn add_full_sync(&mut self) { + let mut partitions = self.syncer.data.replication.sync_partitions(); + info!( + "{}: Adding full sync for ack layout version {}", + F::TABLE_NAME, + partitions.layout_version + ); - let i = rand::thread_rng().gen_range(0..self.todo.len()); - if i == self.todo.len() - 1 { - self.todo.pop() - } else { - let replacement = self.todo.pop().unwrap(); - let ret = std::mem::replace(&mut self.todo[i], replacement); - Some(ret) - } + partitions.partitions.shuffle(&mut thread_rng()); + self.todo = Some(partitions); + self.next_full_sync = Instant::now() + ANTI_ENTROPY_INTERVAL; } } @@ -572,14 +526,48 @@ impl Worker for SyncWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.todo.len() as u64), + queue_length: Some(self.todo.as_ref().map(|x| x.partitions.len()).unwrap_or(0) as u64), ..Default::default() } } async fn work(&mut self, must_exit: &mut watch::Receiver) -> Result { - if let Some(partition) = self.pop_task() { - self.syncer.sync_partition(&partition, must_exit).await?; + self.check_add_full_sync(); + + if let Some(todo) = &mut self.todo { + let partition = todo.partitions.pop().unwrap(); + + // process partition + if let Err(e) = self.syncer.sync_partition(&partition, must_exit).await { + error!( + "{}: Failed to sync partition {:?}: {}", + F::TABLE_NAME, + partition, + e + ); + // if error, put partition back at the other side of the queue, + // so that other partitions will be tried in the meantime + todo.partitions.insert(0, partition); + // TODO: returning an error here will cause the background job worker + // to delay this task for some time, but maybe we don't want to + // delay it if there are lots of failures from nodes that are gone + // (we also don't want zero delays as that will cause lots of useless retries) + return Err(e); + } + + if todo.partitions.is_empty() { + info!( + "{}: Completed full sync for ack layout version {}", + F::TABLE_NAME, + todo.layout_version + ); + self.syncer + .system + .layout_manager + .sync_table_until(F::TABLE_NAME, todo.layout_version); + self.todo = None; + } + Ok(WorkerState::Busy) } else { Ok(WorkerState::Idle) @@ -593,22 +581,16 @@ impl Worker for SyncWorker { self.add_full_sync(); } }, - _ = self.ring_recv.changed() => { - let new_ring = self.ring_recv.borrow(); - if !Arc::ptr_eq(&new_ring, &self.ring) { - self.ring = new_ring.clone(); - drop(new_ring); - debug!("({}) Ring changed, adding full sync to syncer todo list", F::TABLE_NAME); - self.add_full_sync(); - } + _ = self.layout_notify.notified() => { + self.check_add_full_sync(); }, _ = tokio::time::sleep_until(self.next_full_sync.into()) => { self.add_full_sync(); } } - match self.todo.is_empty() { - false => WorkerState::Busy, - true => WorkerState::Idle, + match self.todo.is_some() { + true => WorkerState::Busy, + false => WorkerState::Idle, } } } diff --git a/src/table/table.rs b/src/table/table.rs index 7ad79677..c96f4731 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -2,7 +2,6 @@ use std::borrow::Borrow; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; -use async_trait::async_trait; use futures::stream::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -20,6 +19,7 @@ use garage_util::error::Error; use garage_util::metrics::RecordDuration; use garage_util::migrate::Migrate; +use garage_rpc::rpc_helper::QuorumSetResultTracker; use garage_rpc::system::System; use garage_rpc::*; @@ -80,6 +80,8 @@ impl Table { let syncer = TableSyncer::new(system.clone(), data.clone(), merkle_updater.clone()); let gc = TableGc::new(system.clone(), data.clone()); + system.layout_manager.add_table(F::TABLE_NAME); + let table = Arc::new(Self { system, data, @@ -117,16 +119,16 @@ impl Table { async fn insert_internal(&self, e: &F::E) -> Result<(), Error> { let hash = e.partition_key().hash(); - let who = self.data.replication.write_nodes(&hash); + let who = self.data.replication.write_sets(&hash); let e_enc = Arc::new(ByteBuf::from(e.encode()?)); let rpc = TableRpc::::Update(vec![e_enc]); self.system - .rpc - .try_call_many( + .rpc_helper() + .try_write_many_sets( &self.endpoint, - &who[..], + who.as_ref(), rpc, RequestStrategy::with_priority(PRIO_NORMAL) .with_quorum(self.data.replication.write_quorum()), @@ -141,7 +143,7 @@ impl Table { self.data.queue_insert(tx, e) } - pub async fn insert_many(&self, entries: I) -> Result<(), Error> + pub async fn insert_many(self: &Arc, entries: I) -> Result<(), Error> where I: IntoIterator + Send + Sync, IE: Borrow + Send + Sync, @@ -159,51 +161,127 @@ impl Table { Ok(()) } - async fn insert_many_internal(&self, entries: I) -> Result<(), Error> + async fn insert_many_internal(self: &Arc, entries: I) -> Result<(), Error> where I: IntoIterator + Send + Sync, IE: Borrow + Send + Sync, { - let mut call_list: HashMap<_, Vec<_>> = HashMap::new(); + // The different items will have to be stored on possibly different nodes. + // We will here batch all items into a single request for each concerned + // node, with all of the entries it must store within that request. + // Each entry has to be saved to a specific list of "write sets", i.e. a set + // of node within which a quorum must be achieved. In normal operation, there + // is a single write set which corresponds to the quorum in the current + // cluster layout, but when the layout is updated, multiple write sets might + // have to be handled at once. Here, since we are sending many entries, we + // will have to handle many write sets in all cases. The algorithm is thus + // to send one request to each node with all the items it must save, + // and keep track of the OK responses within each write set: if for all sets + // a quorum of nodes has answered OK, then the insert has succeeded and + // consistency properties (read-after-write) are preserved. + let quorum = self.data.replication.write_quorum(); + + // Serialize all entries and compute the write sets for each of them. + // In the case of sharded table replication, this also takes an "ack lock" + // to the layout manager to avoid ack'ing newer versions which are not + // taken into account by writes in progress (the ack can happen later, once + // all writes that didn't take the new layout into account are finished). + // These locks are released when entries_vec is dropped, i.e. when this + // function returns. + let mut entries_vec = Vec::new(); for entry in entries.into_iter() { let entry = entry.borrow(); let hash = entry.partition_key().hash(); - let who = self.data.replication.write_nodes(&hash); + let mut write_sets = self.data.replication.write_sets(&hash); + for set in write_sets.as_mut().iter_mut() { + // Sort nodes in each write sets to merge write sets with same + // nodes but in possibly different orders + set.sort(); + } let e_enc = Arc::new(ByteBuf::from(entry.encode()?)); - for node in who { - call_list.entry(node).or_default().push(e_enc.clone()); + entries_vec.push((write_sets, e_enc)); + } + + if entries_vec.is_empty() { + return Ok(()); + } + + // Compute a deduplicated list of all of the write sets, + // and compute an index from each node to the position of the sets in which + // it takes part, to optimize the detection of a quorum. + let mut write_sets = entries_vec + .iter() + .flat_map(|(wss, _)| wss.as_ref().iter().map(|ws| ws.as_slice())) + .collect::>(); + write_sets.sort(); + write_sets.dedup(); + + let mut result_tracker = QuorumSetResultTracker::new(&write_sets, quorum); + + // Build a map of all nodes to the entries that must be sent to that node. + let mut call_list: HashMap> = HashMap::new(); + for (write_sets, entry_enc) in entries_vec.iter() { + for write_set in write_sets.as_ref().iter() { + for node in write_set.iter() { + let node_entries = call_list.entry(*node).or_default(); + match node_entries.last() { + Some(x) if Arc::ptr_eq(x, entry_enc) => { + // skip if entry already in list to send to this node + // (could happen if node is in several write sets for this entry) + } + _ => { + node_entries.push(entry_enc.clone()); + } + } + } } } - let call_futures = call_list.drain().map(|(node, entries)| async move { - let rpc = TableRpc::::Update(entries); - - let resp = self - .system - .rpc - .call( - &self.endpoint, - node, - rpc, - RequestStrategy::with_priority(PRIO_NORMAL), - ) - .await?; - Ok::<_, Error>((node, resp)) + // Build futures to actually perform each of the corresponding RPC calls + let call_futures = call_list.into_iter().map(|(node, entries)| { + let this = self.clone(); + async move { + let rpc = TableRpc::::Update(entries); + let resp = this + .system + .rpc_helper() + .call( + &this.endpoint, + node, + rpc, + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(quorum), + ) + .await; + (node, resp) + } }); - let mut resps = call_futures.collect::>(); - let mut errors = vec![]; - while let Some(resp) = resps.next().await { - if let Err(e) = resp { - errors.push(e); + // Run all requests in parallel thanks to FuturesUnordered, and collect results. + let mut resps = call_futures.collect::>(); + + while let Some((node, resp)) = resps.next().await { + result_tracker.register_result(node, resp.map(|_| ())); + + if result_tracker.all_quorums_ok() { + // Success + + // Continue all other requests in background + tokio::spawn(async move { + resps.collect::)>>().await; + }); + + return Ok(()); + } + + if result_tracker.too_many_failures() { + // Too many errors in this set, we know we won't get a quorum + break; } } - if errors.len() > self.data.replication.max_write_errors() { - Err(Error::Message("Too many errors".into())) - } else { - Ok(()) - } + + // Failure, could not get quorum within at least one set + Err(result_tracker.quorum_error()) } pub async fn get( @@ -236,14 +314,13 @@ impl Table { let rpc = TableRpc::::ReadEntry(partition_key.clone(), sort_key.clone()); let resps = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, rpc, RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(self.data.replication.read_quorum()) - .interrupt_after_quorum(true), + .with_quorum(self.data.replication.read_quorum()), ) .await?; @@ -332,14 +409,13 @@ impl Table { let resps = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, rpc, RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(self.data.replication.read_quorum()) - .interrupt_after_quorum(true), + .with_quorum(self.data.replication.read_quorum()), ) .await?; @@ -411,7 +487,7 @@ impl Table { async fn repair_on_read(&self, who: &[Uuid], what: F::E) -> Result<(), Error> { let what_enc = Arc::new(ByteBuf::from(what.encode()?)); self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, who, @@ -423,7 +499,6 @@ impl Table { } } -#[async_trait] impl EndpointHandler> for Table { async fn handle( self: &Arc, diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 2efb0270..46fa6590 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -15,43 +15,40 @@ path = "lib.rs" [dependencies] garage_db.workspace = true +garage_net.workspace = true -arc-swap = "1.0" -async-trait = "0.1" -blake2 = "0.10" -bytes = "1.0" -bytesize = "1.2" -digest = "0.10" -err-derive = "0.3" -hexdump = "0.1" -xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } -hex = "0.4" -lazy_static = "1.4" -tracing = "0.1" -rand = "0.8" -sha2 = "0.10" +arc-swap.workspace = true +async-trait.workspace = true +blake2.workspace = true +bytesize.workspace = true +thiserror.workspace = true +hexdump.workspace = true +xxhash-rust.workspace = true +hex.workspace = true +lazy_static.workspace = true +tracing.workspace = true +rand.workspace = true +sha2.workspace = true -chrono = "0.4" -rmp-serde = "0.15" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_json = "1.0" -toml = "0.6" +chrono.workspace = true +rmp-serde.workspace = true +serde.workspace = true +serde_json.workspace = true +toml.workspace = true -futures = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +tokio.workspace = true -netapp = "0.5" +http.workspace = true +hyper.workspace = true -http = "0.2" -hyper = "0.14" - -opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +opentelemetry.workspace = true [build-dependencies] -rustc_version = "0.4.0" +rustc_version.workspace = true [dev-dependencies] -mktemp = "0.5" +mktemp.workspace = true [features] k2v = [] diff --git a/src/util/async_hash.rs b/src/util/async_hash.rs deleted file mode 100644 index 5631ea6b..00000000 --- a/src/util/async_hash.rs +++ /dev/null @@ -1,61 +0,0 @@ -use bytes::Bytes; -use digest::Digest; - -use tokio::sync::mpsc; -use tokio::task::JoinHandle; - -use crate::data::*; - -/// Compute the sha256 of a slice, -/// spawning on a tokio thread for CPU-intensive processing -/// The argument has to be an owned Bytes, as it is moved out to a new thread. -pub async fn async_sha256sum(data: Bytes) -> Hash { - tokio::task::spawn_blocking(move || sha256sum(&data)) - .await - .unwrap() -} - -/// Compute the blake2sum of a slice, -/// spawning on a tokio thread for CPU-intensive processing. -/// The argument has to be an owned Bytes, as it is moved out to a new thread. -pub async fn async_blake2sum(data: Bytes) -> Hash { - tokio::task::spawn_blocking(move || blake2sum(&data)) - .await - .unwrap() -} - -// ---- - -pub struct AsyncHasher { - sendblk: mpsc::Sender, - task: JoinHandle>, -} - -impl AsyncHasher { - pub fn new() -> Self { - let (sendblk, mut recvblk) = mpsc::channel::(1); - let task = tokio::task::spawn_blocking(move || { - let mut digest = D::new(); - while let Some(blk) = recvblk.blocking_recv() { - digest.update(&blk[..]); - } - digest.finalize() - }); - Self { sendblk, task } - } - - pub async fn update(&self, b: Bytes) { - self.sendblk.send(b).await.unwrap(); - } - - pub async fn finalize(self) -> digest::Output { - drop(self.sendblk); - self.task.await.unwrap() - } -} - -impl Default for AsyncHasher { - fn default() -> Self { - Self::new() - } -} diff --git a/src/util/background/worker.rs b/src/util/background/worker.rs index 8165e2cb..3c938b7e 100644 --- a/src/util/background/worker.rs +++ b/src/util/background/worker.rs @@ -14,7 +14,7 @@ use crate::background::{WorkerInfo, WorkerStatus}; use crate::error::Error; use crate::time::now_msec; -// All workers that haven't exited for this time after an exit signal was recieved +// All workers that haven't exited for this time after an exit signal was received // will be interrupted in the middle of whatever they are doing. const EXIT_DEADLINE: Duration = Duration::from_secs(8); @@ -54,7 +54,7 @@ pub trait Worker: Send { async fn work(&mut self, must_exit: &mut watch::Receiver) -> Result; /// Wait for work: await for some task to become available. This future can be interrupted in - /// the middle for any reason, for example if an interrupt signal was recieved. + /// the middle for any reason, for example if an interrupt signal was received. async fn wait_for_work(&mut self) -> WorkerState; } @@ -115,32 +115,39 @@ impl WorkerProcessor { trace!("{} (TID {}): {:?}", worker.worker.name(), worker.task_id, worker.state); // Save worker info - let mut wi = self.worker_info.lock().unwrap(); - match wi.get_mut(&worker.task_id) { - Some(i) => { - i.state = worker.state; - i.status = worker.worker.status(); - i.errors = worker.errors; - i.consecutive_errors = worker.consecutive_errors; - if worker.last_error.is_some() { - i.last_error = worker.last_error.take(); + { + let mut wi = self.worker_info.lock().unwrap(); + match wi.get_mut(&worker.task_id) { + Some(i) => { + i.state = worker.state; + i.status = worker.worker.status(); + i.errors = worker.errors; + i.consecutive_errors = worker.consecutive_errors; + if worker.last_error.is_some() { + i.last_error = worker.last_error.take(); + } + } + None => { + wi.insert(worker.task_id, WorkerInfo { + name: worker.worker.name(), + state: worker.state, + status: worker.worker.status(), + errors: worker.errors, + consecutive_errors: worker.consecutive_errors, + last_error: worker.last_error.take(), + }); } - } - None => { - wi.insert(worker.task_id, WorkerInfo { - name: worker.worker.name(), - state: worker.state, - status: worker.worker.status(), - errors: worker.errors, - consecutive_errors: worker.consecutive_errors, - last_error: worker.last_error.take(), - }); } } if worker.state == WorkerState::Done { info!("Worker {} (TID {}) exited", worker.worker.name(), worker.task_id); } else { + // Yield to the Tokio scheduler between consecutive Busy steps so + // that a worker which never suspends on its own cannot starve other tasks. + if worker.state == WorkerState::Busy { + tokio::task::yield_now().await; + } workers.push(async move { worker.step().await; worker diff --git a/src/util/config.rs b/src/util/config.rs index cf31c87c..eb889ebe 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -1,12 +1,12 @@ //! Contains type and functions related to Garage configuration file use std::convert::TryFrom; -use std::io::Read; use std::net::SocketAddr; use std::path::PathBuf; use serde::{de, Deserialize}; use crate::error::Error; +use crate::socket_address::UnixOrTCPSocketAddress; /// Represent the whole configuration #[derive(Deserialize, Debug, Clone)] @@ -23,6 +23,21 @@ pub struct Config { #[serde(default)] pub data_fsync: bool, + /// Disable automatic scrubbing of the data directory + #[serde(default)] + pub disable_scrub: bool, + + /// Use local timezone + #[serde(default)] + pub use_local_tz: bool, + + /// Optional directory where metadata snapshots will be store + pub metadata_snapshots_dir: Option, + + /// Automatic snapshot interval for metadata + #[serde(default)] + pub metadata_auto_snapshot_interval: Option, + /// Size of data blocks to save to disk #[serde( deserialize_with = "deserialize_capacity", @@ -30,12 +45,25 @@ pub struct Config { )] pub block_size: usize, - /// Replication mode. Supported values: - /// - none, 1 -> no replication - /// - 2 -> 2-way replication - /// - 3 -> 3-way replication - // (we can add more aliases for this later) - pub replication_mode: String, + /// Maximum number of parallel block writes per PUT request + /// Higher values improve throughput but increase memory usage + /// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #[serde(default = "default_block_max_concurrent_writes_per_request")] + pub block_max_concurrent_writes_per_request: usize, + /// Number of replicas. Can be any positive integer, but uneven numbers are more favorable. + /// - 1 for single-node clusters, or to disable replication + /// - 3 is the recommended and supported setting. + #[serde(default)] + pub replication_factor: Option, + + /// Consistency mode for all for requests through this node + /// - Degraded -> Disable read quorum + /// - Dangerous -> Disable read and write quorum + #[serde(default = "default_consistency_mode")] + pub consistency_mode: String, + + /// Legacy option + pub replication_mode: Option, /// Zstd compression level used on data blocks #[serde( @@ -44,22 +72,45 @@ pub struct Config { )] pub compression_level: Option, + /// Maximum amount of block data to buffer in RAM for sending to + /// remote nodes when these nodes are on slower links + #[serde( + deserialize_with = "deserialize_capacity", + default = "default_block_ram_buffer_max" + )] + pub block_ram_buffer_max: usize, + + /// Maximum number of concurrent reads of block files on disk + #[serde(default = "default_block_max_concurrent_reads")] + pub block_max_concurrent_reads: usize, + + /// Skip the permission check of secret files. Useful when + /// POSIX ACLs (or more complex chmods) are used. + #[serde(default)] + pub allow_world_readable_secrets: bool, + /// RPC secret key: 32 bytes hex encoded pub rpc_secret: Option, /// Optional file where RPC secret key is read from - pub rpc_secret_file: Option, - + pub rpc_secret_file: Option, /// Address to bind for RPC pub rpc_bind_addr: SocketAddr, + /// Bind outgoing sockets to rpc_bind_addr's IP address as well + #[serde(default)] + pub rpc_bind_outgoing: bool, /// Public IP address of this node pub rpc_public_addr: Option, - /// Timeout for Netapp's ping messagess + /// In case `rpc_public_addr` was not set, this can filter + /// the addresses announced to other peers to a specific subnet. + pub rpc_public_addr_subnet: Option, + + /// Timeout for Netapp's ping messages pub rpc_ping_timeout_msec: Option, /// Timeout for Netapp RPC calls pub rpc_timeout_msec: Option, - // -- Bootstraping and discovery + // -- Bootstrapping and discovery /// Bootstrap peers RPC address #[serde(default)] pub bootstrap_peers: Vec, @@ -72,24 +123,18 @@ pub struct Config { pub kubernetes_discovery: Option, // -- DB - /// Database engine to use for metadata (options: sled, sqlite, lmdb) + /// Database engine to use for metadata (options: sqlite, lmdb) #[serde(default = "default_db_engine")] pub db_engine: String, - /// Sled cache size, in bytes - #[serde( - deserialize_with = "deserialize_capacity", - default = "default_sled_cache_capacity" - )] - pub sled_cache_capacity: usize, - /// Sled flush interval in milliseconds - #[serde(default = "default_sled_flush_every_ms")] - pub sled_flush_every_ms: u64, - /// LMDB map size #[serde(deserialize_with = "deserialize_capacity", default)] pub lmdb_map_size: usize, + /// Fjall block cache size + #[serde(deserialize_with = "deserialize_capacity", default)] + pub fjall_block_cache_size: usize, + // -- APIs /// Configuration for S3 api pub s3_api: S3ApiConfig, @@ -103,6 +148,10 @@ pub struct Config { /// Configuration for the admin API endpoint #[serde(default = "Default::default")] pub admin: AdminConfig, + + /// Allow punycode in bucket names + #[serde(default)] + pub allow_punycode: bool, } /// Value for data_dir: either a single directory or a list of dirs with attributes @@ -129,7 +178,7 @@ pub struct DataDir { #[derive(Deserialize, Debug, Clone)] pub struct S3ApiConfig { /// Address and port to bind for api serving - pub api_bind_addr: Option, + pub api_bind_addr: Option, /// S3 region to use pub s3_region: String, /// Suffix to remove from domain name to find bucket. If None, @@ -141,33 +190,36 @@ pub struct S3ApiConfig { #[derive(Deserialize, Debug, Clone)] pub struct K2VApiConfig { /// Address and port to bind for api serving - pub api_bind_addr: SocketAddr, + pub api_bind_addr: UnixOrTCPSocketAddress, } /// Configuration for serving files as normal web server #[derive(Deserialize, Debug, Clone)] pub struct WebConfig { /// Address and port to bind for web serving - pub bind_addr: SocketAddr, + pub bind_addr: UnixOrTCPSocketAddress, /// Suffix to remove from domain name to find bucket pub root_domain: String, + /// Whether to add the requested domain to exported Prometheus metrics + #[serde(default)] + pub add_host_to_metrics: bool, } /// Configuration for the admin and monitoring HTTP API #[derive(Deserialize, Debug, Clone, Default)] pub struct AdminConfig { /// Address and port to bind for admin API serving - pub api_bind_addr: Option, + pub api_bind_addr: Option, /// Bearer token to use to scrape metrics pub metrics_token: Option, /// File to read metrics token from - pub metrics_token_file: Option, + pub metrics_token_file: Option, /// Bearer token to use to access Admin API endpoints pub admin_token: Option, /// File to read admin token from - pub admin_token_file: Option, + pub admin_token_file: Option, /// OTLP server to where to export traces pub trace_sink: Option, @@ -220,80 +272,32 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } +pub fn default_block_max_concurrent_writes_per_request() -> usize { + 3 +} +/// Read and parse configuration +pub fn read_config(config_file: PathBuf) -> Result { + let config = std::fs::read_to_string(config_file)?; + + Ok(toml::from_str(&config)?) +} + fn default_db_engine() -> String { "lmdb".into() } -fn default_sled_cache_capacity() -> usize { - 128 * 1024 * 1024 -} -fn default_sled_flush_every_ms() -> u64 { - 2000 -} fn default_block_size() -> usize { 1048576 } - -/// Read and parse configuration -pub fn read_config(config_file: PathBuf) -> Result { - let mut file = std::fs::OpenOptions::new() - .read(true) - .open(config_file.as_path())?; - - let mut config = String::new(); - file.read_to_string(&mut config)?; - - let mut parsed_config: Config = toml::from_str(&config)?; - - secret_from_file( - &mut parsed_config.rpc_secret, - &parsed_config.rpc_secret_file, - "rpc_secret", - )?; - secret_from_file( - &mut parsed_config.admin.metrics_token, - &parsed_config.admin.metrics_token_file, - "admin.metrics_token", - )?; - secret_from_file( - &mut parsed_config.admin.admin_token, - &parsed_config.admin.admin_token_file, - "admin.admin_token", - )?; - - Ok(parsed_config) +fn default_block_ram_buffer_max() -> usize { + 256 * 1024 * 1024 +} +fn default_block_max_concurrent_reads() -> usize { + 16 } -fn secret_from_file( - secret: &mut Option, - secret_file: &Option, - name: &'static str, -) -> Result<(), Error> { - match (&secret, &secret_file) { - (_, None) => { - // no-op - } - (Some(_), Some(_)) => { - return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); - } - (None, Some(file_path)) => { - #[cfg(unix)] - if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") { - use std::os::unix::fs::MetadataExt; - let metadata = std::fs::metadata(file_path)?; - if metadata.mode() & 0o077 != 0 { - return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); - } - } - let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?; - let mut secret_buf = String::new(); - file.read_to_string(&mut secret_buf)?; - // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. - // also editors sometimes add a trailing newline - *secret = Some(String::from(secret_buf.trim_end())); - } - } - Ok(()) +fn default_consistency_mode() -> String { + "consistent".into() } fn default_compression() -> Option { @@ -407,7 +411,7 @@ mod tests { r#" metadata_dir = "/tmp/garage/meta" data_dir = "/tmp/garage/data" - replication_mode = "3" + replication_factor = 3 rpc_bind_addr = "[::]:3901" rpc_secret = "foo" @@ -424,83 +428,4 @@ mod tests { Ok(()) } - - #[test] - fn test_rpc_secret_file_works() -> Result<(), Error> { - let path_secret = mktemp::Temp::new_file()?; - let mut file_secret = File::create(path_secret.as_path())?; - writeln!(file_secret, "foo")?; - drop(file_secret); - - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - let path_secret_path = path_secret.as_path(); - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret_file = "{}" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "#, - path_secret_path.display() - )?; - let config = super::read_config(path_config.to_path_buf())?; - assert_eq!("foo", config.rpc_secret.unwrap()); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let metadata = std::fs::metadata(&path_secret_path)?; - let mut perm = metadata.permissions(); - perm.set_mode(0o660); - std::fs::set_permissions(&path_secret_path, perm)?; - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false"); - assert!(super::read_config(path_config.to_path_buf()).is_err()); - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true"); - assert!(super::read_config(path_config.to_path_buf()).is_ok()); - } - - drop(path_config); - drop(path_secret); - drop(file_config); - Ok(()) - } - - #[test] - fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret= "dummy" - rpc_secret_file = "dummy" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "# - )?; - assert_eq!( - "only one of `rpc_secret` and `rpc_secret_file` can be set", - super::read_config(path_config.to_path_buf()) - .unwrap_err() - .to_string() - ); - drop(path_config); - drop(file_config); - Ok(()) - } } diff --git a/src/util/crdt/crdt.rs b/src/util/crdt/crdt.rs index 06876897..fdf63084 100644 --- a/src/util/crdt/crdt.rs +++ b/src/util/crdt/crdt.rs @@ -33,8 +33,8 @@ pub trait Crdt { /// arises very often, for example with a Lww or a LwwMap: the value type has to be a CRDT so that /// we have a rule for what to do when timestamps aren't enough to disambiguate (in a distributed /// system, anything can happen!), and with AutoCrdt the rule is to make an arbitrary (but -/// determinstic) choice between the two. When using an Option instead with this impl, ambiguity -/// cases are explicitely stored as None, which allows us to detect the ambiguity and handle it in +/// deterministic) choice between the two. When using an Option instead with this impl, ambiguity +/// cases are explicitly stored as None, which allows us to detect the ambiguity and handle it in /// the way we want. (this can only work if we are happy with losing the value when an ambiguity /// arises) impl Crdt for Option diff --git a/src/util/crdt/lww.rs b/src/util/crdt/lww.rs index 958844c9..80747406 100644 --- a/src/util/crdt/lww.rs +++ b/src/util/crdt/lww.rs @@ -16,7 +16,7 @@ use crate::crdt::crdt::*; /// In our case, we add the constraint that the value that is wrapped inside the LWW CRDT must /// itself be a CRDT: in the case when the timestamp does not allow us to decide on which value to /// keep, the merge rule of the inner CRDT is applied on the wrapped values. (Note that all types -/// that implement the `Ord` trait get a default CRDT implemetnation that keeps the maximum value. +/// that implement the `Ord` trait get a default CRDT implementation that keeps the maximum value. /// This enables us to use LWW directly with primitive data types such as numbers or strings. It is /// generally desirable in this case to never explicitly produce LWW values with the same timestamp /// but different inner values, as the rule to keep the maximum value isn't generally the desired @@ -28,9 +28,9 @@ use crate::crdt::crdt::*; /// /// Given that clocks are not too desynchronized, this assumption /// is enough for most cases, as there is few chance that two humans -/// coordonate themself faster than the time difference between two NTP servers. +/// coordinate themself faster than the time difference between two NTP servers. /// -/// As a more concret example, let's suppose you want to upload a file +/// As a more concrete example, let's suppose you want to upload a file /// with the same key (path) in the same bucket at the very same time. /// For each request, the file will be timestamped by the receiving server /// and may differ from what you observed with your atomic clock! @@ -84,16 +84,16 @@ where &self.v } - /// Take the value inside the CRDT (discards the timesamp) + /// Take the value inside the CRDT (discards the timestamp) pub fn take(self) -> T { self.v } /// Get a mutable reference to the CRDT's value /// - /// This is usefull to mutate the inside value without changing the LWW timestamp. + /// This is useful to mutate the inside value without changing the LWW timestamp. /// When such mutation is done, the merge between two LWW values is done using the inner - /// CRDT's merge operation. This is usefull in the case where the inner CRDT is a large + /// CRDT's merge operation. This is useful in the case where the inner CRDT is a large /// data type, such as a map, and we only want to change a single item in the map. /// To do this, we can produce a "CRDT delta", i.e. a LWW that contains only the modification. /// This delta consists in a LWW with the same timestamp, and the map diff --git a/src/util/crdt/lww_map.rs b/src/util/crdt/lww_map.rs index 88113856..def0ebeb 100644 --- a/src/util/crdt/lww_map.rs +++ b/src/util/crdt/lww_map.rs @@ -109,7 +109,7 @@ where } /// Takes all of the values of the map and returns them. The current map is reset to the - /// empty map. This is very usefull to produce in-place a new map that contains only a delta + /// empty map. This is very useful to produce in-place a new map that contains only a delta /// that modifies a certain value: /// /// ```ignore @@ -162,7 +162,7 @@ where } } - /// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values. + /// Gets a reference to all of the items, as a slice. Useful to iterate on all map values. /// In most case you will want to ignore the timestamp (second item of the tuple). pub fn items(&self) -> &[(K, u64, V)] { &self.vals[..] diff --git a/src/util/crdt/map.rs b/src/util/crdt/map.rs index 5d1e1520..adac3c38 100644 --- a/src/util/crdt/map.rs +++ b/src/util/crdt/map.rs @@ -57,7 +57,7 @@ where Err(_) => None, } } - /// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values. + /// Gets a reference to all of the items, as a slice. Useful to iterate on all map values. pub fn items(&self) -> &[(K, V)] { &self.vals[..] } diff --git a/src/util/data.rs b/src/util/data.rs index bdd8daee..1fe7dfe0 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -83,17 +83,30 @@ impl FixedBytes32 { ret.copy_from_slice(by); Some(Self(ret)) } + /// Return the next hash + pub fn increment(&self) -> Option { + let mut ret = *self; + for byte in ret.0.iter_mut().rev() { + if *byte == u8::MAX { + *byte = 0; + } else { + *byte = *byte + 1; + return Some(ret); + } + } + return None; + } } -impl From for FixedBytes32 { - fn from(node_id: netapp::NodeID) -> FixedBytes32 { +impl From for FixedBytes32 { + fn from(node_id: garage_net::NodeID) -> FixedBytes32 { FixedBytes32::try_from(node_id.as_ref()).unwrap() } } -impl From for netapp::NodeID { - fn from(bytes: FixedBytes32) -> netapp::NodeID { - netapp::NodeID::from_slice(bytes.as_slice()).unwrap() +impl From for garage_net::NodeID { + fn from(bytes: FixedBytes32) -> garage_net::NodeID { + garage_net::NodeID::from_slice(bytes.as_slice()).unwrap() } } @@ -140,3 +153,25 @@ pub fn fasthash(data: &[u8]) -> FastHash { pub fn gen_uuid() -> Uuid { rand::thread_rng().gen::<[u8; 32]>().into() } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_increment() { + let zero: FixedBytes32 = [0u8; 32].into(); + let mut one: FixedBytes32 = [0u8; 32].into(); + one.0[31] = 1; + let max: FixedBytes32 = [0xFFu8; 32].into(); + assert_eq!(zero.increment(), Some(one)); + assert_eq!(max.increment(), None); + + let mut test: FixedBytes32 = [0u8; 32].into(); + let i = 0x198DF97209F8FFFFu64; + test.0[24..32].copy_from_slice(&u64::to_be_bytes(i)); + let mut test2: FixedBytes32 = [0u8; 32].into(); + test2.0[24..32].copy_from_slice(&u64::to_be_bytes(i + 1)); + assert_eq!(test.increment(), Some(test2)); + } +} diff --git a/src/util/encode.rs b/src/util/encode.rs index a9ab9a35..c6815d49 100644 --- a/src/util/encode.rs +++ b/src/util/encode.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; -/// Serialize to MessagePacki, without versionning -/// (see garage_util::migrate for functions that manage versionned +/// Serialize to MessagePack, without versioning +/// (see garage_util::migrate for functions that manage versioned /// data formats) pub fn nonversioned_encode(val: &T) -> Result, rmp_serde::encode::Error> where @@ -13,8 +13,8 @@ where Ok(wr) } -/// Deserialize from MessagePacki, without versionning -/// (see garage_util::migrate for functions that manage versionned +/// Deserialize from MessagePack, without versioning +/// (see garage_util::migrate for functions that manage versioned /// data formats) pub fn nonversioned_decode(bytes: &[u8]) -> Result where diff --git a/src/util/error.rs b/src/util/error.rs index 3fcee71d..170d2687 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -2,7 +2,7 @@ use std::fmt; use std::io; -use err_derive::Error; +use thiserror::Error; use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; @@ -12,64 +12,61 @@ use crate::encode::debug_serialize; /// Regroup all Garage errors #[derive(Debug, Error)] pub enum Error { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] io::Error), + #[error("IO error: {0}")] + Io(#[from] io::Error), - #[error(display = "Hyper error: {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Hyper error: {0}")] + Hyper(#[from] hyper::Error), - #[error(display = "HTTP error: {}", _0)] - Http(#[error(source)] http::Error), + #[error("HTTP error: {0}")] + Http(#[from] http::Error), - #[error(display = "Invalid HTTP header value: {}", _0)] - HttpHeader(#[error(source)] http::header::ToStrError), + #[error("Invalid HTTP header value: {0}")] + HttpHeader(#[from] http::header::ToStrError), - #[error(display = "Netapp error: {}", _0)] - Netapp(#[error(source)] netapp::error::Error), + #[error("Network error: {0}")] + Net(#[from] garage_net::error::Error), - #[error(display = "DB error: {}", _0)] - Db(#[error(source)] garage_db::Error), + #[error("DB error: {0}")] + Db(#[from] garage_db::Error), - #[error(display = "Messagepack encode error: {}", _0)] - RmpEncode(#[error(source)] rmp_serde::encode::Error), - #[error(display = "Messagepack decode error: {}", _0)] - RmpDecode(#[error(source)] rmp_serde::decode::Error), - #[error(display = "JSON error: {}", _0)] - Json(#[error(source)] serde_json::error::Error), - #[error(display = "TOML decode error: {}", _0)] - TomlDecode(#[error(source)] toml::de::Error), + #[error("Messagepack encode error: {0}")] + RmpEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RmpDecode(#[from] rmp_serde::decode::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::error::Error), + #[error("TOML decode error: {0}")] + TomlDecode(#[from] toml::de::Error), - #[error(display = "Tokio join error: {}", _0)] - TokioJoin(#[error(source)] tokio::task::JoinError), + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), - #[error(display = "Tokio semaphore acquire error: {}", _0)] - TokioSemAcquire(#[error(source)] tokio::sync::AcquireError), + #[error("Tokio semaphore acquire error: {0}")] + TokioSemAcquire(#[from] tokio::sync::AcquireError), - #[error(display = "Tokio broadcast receive error: {}", _0)] - TokioBcastRecv(#[error(source)] tokio::sync::broadcast::error::RecvError), + #[error("Tokio broadcast receive error: {0}")] + TokioBcastRecv(#[from] tokio::sync::broadcast::error::RecvError), - #[error(display = "Remote error: {}", _0)] + #[error("Remote error: {0}")] RemoteError(String), - #[error(display = "Timeout")] + #[error("Timeout")] Timeout, - #[error( - display = "Could not reach quorum of {}. {} of {} request succeeded, others returned errors: {:?}", - _0, - _1, - _2, - _3 - )] - Quorum(usize, usize, usize, Vec), + #[error("Could not reach quorum of {0} (sets={1:?}). {2} of {3} request succeeded, others returned errors: {4:?}")] + Quorum(usize, Option, usize, usize, Vec), - #[error(display = "Unexpected RPC message: {}", _0)] + #[error("Unexpected RPC message: {0}")] UnexpectedRpcMessage(String), - #[error(display = "Corrupt data: does not match hash {:?}", _0)] + #[error("Corrupt data: does not match hash {0:?}")] CorruptData(Hash), - #[error(display = "{}", _0)] + #[error("Missing block {0:?}: no node returned a valid block")] + MissingBlock(Hash), + + #[error("{0}")] Message(String), } diff --git a/src/util/lib.rs b/src/util/lib.rs index 15f0f829..8b035ff0 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -3,7 +3,6 @@ #[macro_use] extern crate tracing; -pub mod async_hash; pub mod background; pub mod config; pub mod crdt; @@ -14,6 +13,7 @@ pub mod forwarded_headers; pub mod metrics; pub mod migrate; pub mod persister; +pub mod socket_address; pub mod time; pub mod tranquilizer; pub mod version; diff --git a/src/util/socket_address.rs b/src/util/socket_address.rs new file mode 100644 index 00000000..f01225f6 --- /dev/null +++ b/src/util/socket_address.rs @@ -0,0 +1,44 @@ +use std::fmt::{Debug, Display, Formatter}; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::str::FromStr; + +use serde::de::Error; +use serde::{Deserialize, Deserializer}; + +#[derive(Debug, Clone)] +pub enum UnixOrTCPSocketAddress { + TCPSocket(SocketAddr), + UnixSocket(PathBuf), +} + +impl Display for UnixOrTCPSocketAddress { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + UnixOrTCPSocketAddress::TCPSocket(address) => write!(formatter, "http://{}", address), + UnixOrTCPSocketAddress::UnixSocket(path) => { + write!(formatter, "http+unix://{}", path.to_string_lossy()) + } + } + } +} + +impl<'de> Deserialize<'de> for UnixOrTCPSocketAddress { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let string = String::deserialize(deserializer)?; + let string = string.as_str(); + + if string.starts_with("/") { + Ok(UnixOrTCPSocketAddress::UnixSocket( + PathBuf::from_str(string).map_err(Error::custom)?, + )) + } else { + Ok(UnixOrTCPSocketAddress::TCPSocket( + SocketAddr::from_str(string).map_err(Error::custom)?, + )) + } + } +} diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 6d0eba3a..e0cb317f 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.8.4" +version = "1.3.1" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" @@ -14,18 +14,20 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api.workspace = true +garage_api_common.workspace = true +garage_api_s3.workspace = true garage_model.workspace = true garage_util.workspace = true garage_table.workspace = true -err-derive = "0.3" -tracing = "0.1" -percent-encoding = "2.1.0" +thiserror.workspace = true +tracing.workspace = true +percent-encoding.workspace = true -futures = "0.3" +http.workspace = true +http-body-util.workspace = true +hyper.workspace = true -http = "0.2" -hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } +tokio.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true diff --git a/src/web/error.rs b/src/web/error.rs index bd8f17b5..aef74923 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -1,31 +1,31 @@ -use err_derive::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; -use garage_api::generic_server::ApiError; +use garage_api_common::generic_server::ApiError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { /// An error received from the API crate - #[error(display = "API error: {}", _0)] - ApiError(garage_api::s3::error::Error), + #[error("API error: {0}")] + ApiError(garage_api_s3::error::Error), /// The file does not exist - #[error(display = "Not found")] + #[error("Not found")] NotFound, /// The client sent a request without host, or with unsupported method - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), } impl From for Error where - garage_api::s3::error::Error: From, + garage_api_s3::error::Error: From, { fn from(err: T) -> Self { - Error::ApiError(garage_api::s3::error::Error::from(err)) + Error::ApiError(garage_api_s3::error::Error::from(err)) } } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 287aef1a..ea02ab0f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,12 +1,14 @@ -use std::{convert::Infallible, net::SocketAddr, sync::Arc}; +use std::fs::{self, Permissions}; +use std::os::unix::prelude::PermissionsExt; +use std::sync::Arc; -use futures::future::Future; +use tokio::net::{TcpListener, UnixListener}; +use tokio::sync::watch; use hyper::{ - header::{HeaderValue, HOST}, - server::conn::AddrStream, - service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, StatusCode, + body::Incoming as IncomingBody, + header::{HeaderValue, HOST, LOCATION}, + Method, Request, Response, StatusCode, }; use opentelemetry::{ @@ -18,20 +20,26 @@ use opentelemetry::{ use crate::error::*; -use garage_api::helpers::{authority_to_host, host_to_bucket}; -use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket}; -use garage_api::s3::error::{ +use garage_api_common::cors::{ + add_cors_headers, find_matching_cors_rule, handle_options_for_bucket, +}; +use garage_api_common::generic_server::{server_loop, UnixListenerOn}; +use garage_api_common::helpers::*; +use garage_api_s3::error::{ CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError, }; -use garage_api::s3::get::{handle_get, handle_head}; +use garage_api_s3::get::{handle_get_without_ctx, handle_head_without_ctx}; +use garage_api_s3::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; use garage_model::garage::Garage; use garage_table::*; +use garage_util::config::WebConfig; use garage_util::data::Uuid; use garage_util::error::Error as GarageError; use garage_util::forwarded_headers; use garage_util::metrics::{gen_trace_id, RecordDuration}; +use garage_util::socket_address::UnixOrTCPSocketAddress; struct WebMetrics { request_counter: Counter, @@ -63,61 +71,80 @@ pub struct WebServer { garage: Arc, metrics: Arc, root_domain: String, + add_host_to_metrics: bool, } impl WebServer { /// Run a web server - pub async fn run( - garage: Arc, - addr: SocketAddr, - root_domain: String, - shutdown_signal: impl Future, - ) -> Result<(), GarageError> { + pub fn new(garage: Arc, config: &WebConfig) -> Arc { let metrics = Arc::new(WebMetrics::new()); - let web_server = Arc::new(WebServer { + Arc::new(WebServer { garage, metrics, - root_domain, - }); + root_domain: config.root_domain.clone(), + add_host_to_metrics: config.add_host_to_metrics, + }) + } - let service = make_service_fn(|conn: &AddrStream| { - let web_server = web_server.clone(); + pub async fn run( + self: Arc, + bind_addr: UnixOrTCPSocketAddress, + must_exit: watch::Receiver, + ) -> Result<(), GarageError> { + let server_name = "Web".into(); + info!("Web server listening on {}", bind_addr); - let client_addr = conn.remote_addr(); - async move { - Ok::<_, Error>(service_fn(move |req: Request| { - let web_server = web_server.clone(); + match bind_addr { + UnixOrTCPSocketAddress::TCPSocket(addr) => { + let listener = TcpListener::bind(addr).await?; - web_server.handle_request(req, client_addr) - })) + let handler = + move |stream, socketaddr| self.clone().handle_request(stream, socketaddr); + server_loop(server_name, listener, handler, must_exit).await } - }); + UnixOrTCPSocketAddress::UnixSocket(ref path) => { + if path.exists() { + fs::remove_file(path)? + } - let server = Server::bind(&addr).serve(service); - let graceful = server.with_graceful_shutdown(shutdown_signal); - info!("Web server listening on http://{}", addr); + let listener = UnixListener::bind(path)?; + let listener = UnixListenerOn(listener, path.display().to_string()); - graceful.await?; - Ok(()) + fs::set_permissions(path, Permissions::from_mode(0o222))?; + + let handler = + move |stream, socketaddr| self.clone().handle_request(stream, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + } } async fn handle_request( self: Arc, - req: Request, - addr: SocketAddr, - ) -> Result, Infallible> { + req: Request, + addr: String, + ) -> Result>, http::Error> { + let host_header = req + .headers() + .get(HOST) + .and_then(|x| x.to_str().ok()) + .unwrap_or("") + .to_string(); + if let Ok(forwarded_for_ip_addr) = forwarded_headers::handle_forwarded_for_headers(req.headers()) { + // uri() below has a preceding '/', so no space with host info!( - "{} (via {}) {} {}", + "{} (via {}) {} {}{}", forwarded_for_ip_addr, addr, req.method(), + host_header, req.uri() ); } else { - info!("{} {} {}", addr, req.method(), req.uri()); + info!("{} {} {}{}", addr, req.method(), host_header, req.uri()); } // Lots of instrumentation @@ -126,12 +153,18 @@ impl WebServer { .span_builder(format!("Web {} request", req.method())) .with_trace_id(gen_trace_id()) .with_attributes(vec![ + KeyValue::new("host", format!("{}", host_header.clone())), KeyValue::new("method", format!("{}", req.method())), KeyValue::new("uri", req.uri().to_string()), ]) .start(&tracer); - let metrics_tags = &[KeyValue::new("method", req.method().to_string())]; + let mut metrics_tags = vec![KeyValue::new("method", req.method().to_string())]; + if self.add_host_to_metrics { + metrics_tags.push(KeyValue::new("host", host_header.clone())); + } + + let req = req.map(|_| ()); // The actual handler let res = self @@ -146,24 +179,30 @@ impl WebServer { // Returning the result match res { Ok(res) => { - debug!("{} {} {}", req.method(), res.status(), req.uri()); - Ok(res) + debug!( + "{} {} {}{}", + req.method(), + res.status(), + host_header, + req.uri() + ); + Ok(res + .map(|body| BoxBody::new(http_body_util::BodyExt::map_err(body, Error::from)))) } Err(error) => { info!( - "{} {} {} {}", + "{} {} {}{} {}", req.method(), error.http_status_code(), + host_header, req.uri(), error ); - self.metrics.error_counter.add( - 1, - &[ - metrics_tags[0].clone(), - KeyValue::new("status_code", error.http_status_code().to_string()), - ], - ); + metrics_tags.push(KeyValue::new( + "status_code", + error.http_status_code().to_string(), + )); + self.metrics.error_counter.add(1, &metrics_tags); Ok(error_to_res(error)) } } @@ -180,7 +219,10 @@ impl WebServer { Ok(exists) } - async fn serve_file(self: &Arc, req: &Request) -> Result, Error> { + async fn serve_file( + self: &Arc, + req: &Request<()>, + ) -> Result>, Error> { // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() @@ -203,14 +245,13 @@ impl WebServer { // Check bucket isn't deleted and has website access enabled let bucket = self .garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NotFound)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await + .map_err(|_| Error::NotFound)?; + let bucket_params = bucket.state.into_option().unwrap(); - let website_config = bucket - .params() - .ok_or(Error::NotFound)? + let website_config = bucket_params .website_config .get() .as_ref() @@ -227,9 +268,23 @@ impl WebServer { ); let ret_doc = match *req.method() { - Method::OPTIONS => handle_options_for_bucket(req, &bucket), - Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await, - Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await, + Method::OPTIONS => handle_options_for_bucket(req, &bucket_params) + .map_err(ApiError::from) + .map(|res| res.map(|_empty_body: EmptyBody| empty_body())), + Method::HEAD => { + handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await + } + Method::GET => { + handle_get_without_ctx( + self.garage.clone(), + req, + bucket_id, + &key, + None, + Default::default(), + ) + .await + } _ => Err(ApiError::bad_request("HTTP method not supported")), }; @@ -240,8 +295,16 @@ impl WebServer { { Ok(Response::builder() .status(StatusCode::FOUND) - .header("Location", url) - .body(Body::empty()) + .header(LOCATION, url) + .body(empty_body()) + .unwrap()) + } + (Ok(ret), _) if ret.headers().contains_key(X_AMZ_WEBSITE_REDIRECT_LOCATION) => { + let redirect_location = ret.headers().get(X_AMZ_WEBSITE_REDIRECT_LOCATION).unwrap(); + Ok(Response::builder() + .status(StatusCode::MOVED_PERMANENTLY) + .header(LOCATION, redirect_location) + .body(empty_body()) .unwrap()) } _ => ret_doc, @@ -270,10 +333,18 @@ impl WebServer { // Create a fake HTTP request with path = the error document let req2 = Request::builder() .uri(format!("http://{}/{}", host, &error_document)) - .body(Body::empty()) + .body(()) .unwrap(); - match handle_get(self.garage.clone(), &req2, bucket_id, &error_document, None).await + match handle_get_without_ctx( + self.garage.clone(), + &req2, + bucket_id, + &error_document, + None, + Default::default(), + ) + .await { Ok(mut error_doc) => { // The error won't be logged back in handle_request, @@ -308,7 +379,7 @@ impl WebServer { } Ok(mut resp) => { // Maybe add CORS headers - if let Some(rule) = find_matching_cors_rule(&bucket, req)? { + if let Some(rule) = find_matching_cors_rule(&bucket_params, req)? { add_cors_headers(&mut resp, rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } @@ -318,7 +389,7 @@ impl WebServer { } } -fn error_to_res(e: Error) -> Response { +fn error_to_res(e: Error) -> Response> { // If we are here, it is either that: // - there was an error before trying to get the requested URL // from the bucket (e.g. bucket not found) @@ -326,10 +397,30 @@ fn error_to_res(e: Error) -> Response { // was a HEAD request or we couldn't get the error document) // We do NOT enter this code path when returning the bucket's // error document (this is handled in serve_file) - let body = Body::from(format!("{}\n", e)); - let mut http_error = Response::new(body); + let mut body_str = format!( + r"{http_code} {code_text} +

{http_code} {code_text}

", + http_code = e.http_status_code().as_u16(), + code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"), + ); + if let Error::ApiError(ref err) = e { + body_str.push_str(&format!( + r" +
    +
  • Code: {s3_code}
  • +
  • Message: {s3_message}.
  • +
", + s3_code = err.aws_code(), + s3_message = err, + )); + } + let mut http_error = Response::new(string_body(body_str)); *http_error.status_mut() = e.http_status_code(); e.add_headers(http_error.headers_mut()); + http_error.headers_mut().insert( + http::header::CONTENT_TYPE, + "text/html; charset=utf-8".parse().unwrap(), + ); http_error }