diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index c219c414..00000000 --- a/.drone.yml +++ /dev/null @@ -1,288 +0,0 @@ ---- -kind: pipeline -name: default - -node: - nix-daemon: 1 - -steps: - - name: check formatting - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr rust --run "cargo fmt -- --check" - - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - - name: unit + func tests - image: nixpkgs/nix:nixos-22.05 - environment: - GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-build --no-build-output --attr test.amd64 - - ./result/bin/garage_db-* - - ./result/bin/garage_api-* - - ./result/bin/garage_model-* - - ./result/bin/garage_rpc-* - - ./result/bin/garage_table-* - - ./result/bin/garage_util-* - - ./result/bin/garage_web-* - - ./result/bin/garage-* - - ./result/bin/integration-* - - rm result - - - name: integration tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - -trigger: - event: - - custom - - push - - pull_request - - tag - - cron - ---- -kind: pipeline -type: docker -name: release-linux-amd64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "x86_64-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/amd64" - CONTAINER_NAME: "dxflrs/amd64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-i386 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "i686-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/386" - CONTAINER_NAME: "dxflrs/386_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "aarch64-unknown-linux-musl" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm64" - CONTAINER_NAME: "dxflrs/arm64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "armv6l-unknown-linux-musleabihf" - commands: - - nix-shell --attr release --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm" - CONTAINER_NAME: "dxflrs/arm_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: refresh-release-page - -node: - nix-daemon: 1 - -steps: - - name: multiarch-docker - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - HOME: "/root" - commands: - - mkdir -p /root/.docker - - echo $DOCKER_AUTH > /root/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "multiarch_docker" - - name: refresh-index - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - commands: - - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --attr release --run "refresh_index" - -depends_on: - - release-linux-amd64 - - release-linux-i386 - - release-linux-arm64 - - release-linux-arm - -trigger: - event: - - promote - - cron - ---- -kind: signature -hmac: ac09a5a8c82502f67271f93afa1e1e21ce66383b8e24a6deb26b285cc1c378ba - -... diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml new file mode 100644 index 00000000..4dc7d3c9 --- /dev/null +++ b/.woodpecker/debug.yaml @@ -0,0 +1,44 @@ +labels: + nix: "enabled" + +when: + event: + - push + - tag + - pull_request + - deployment + - cron + - manual + +steps: + - name: check formatting + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.fmt + + - name: build + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.dev + + - name: unit + func tests (lmdb) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-lmdb + + - name: unit + func tests (sqlite) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-sqlite + + - name: unit + func tests (fjall) + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.tests-fjall + + - name: integration tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build -j4 --attr flakePackages.dev + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + depends_on: [ build ] diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml new file mode 100644 index 00000000..8f3b482f --- /dev/null +++ b/.woodpecker/publish.yaml @@ -0,0 +1,33 @@ +labels: + nix: "enabled" + +when: + event: + - deployment + - cron + +depends_on: + - release + +steps: + - name: refresh-index + image: nixpkgs/nix:nixos-24.05 + environment: + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf + - nix-shell --attr ci --run "refresh_index" + + - name: multiarch-docker + image: nixpkgs/nix:nixos-24.05 + environment: + DOCKER_AUTH: + from_secret: docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "multiarch_docker" diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml new file mode 100644 index 00000000..a94a9ccf --- /dev/null +++ b/.woodpecker/release.yaml @@ -0,0 +1,71 @@ +labels: + nix: "enabled" + +when: + event: + - deployment + - cron + +matrix: + include: + - ARCH: amd64 + TARGET: x86_64-unknown-linux-musl + - ARCH: i386 + TARGET: i686-unknown-linux-musl + - ARCH: arm64 + TARGET: aarch64-unknown-linux-musl + - ARCH: arm + TARGET: armv6l-unknown-linux-musleabihf + +steps: + - name: build + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + + - name: check is static binary + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage" + + - name: integration tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + when: + - matrix: + ARCH: amd64 + - matrix: + ARCH: i386 + + - name: upgrade tests + image: nixpkgs/nix:nixos-24.05 + commands: + - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) + when: + - matrix: + ARCH: amd64 + + - name: push static binary + image: nixpkgs/nix:nixos-24.05 + environment: + TARGET: "${TARGET}" + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - nix-shell --attr ci --run "to_s3" + + - name: docker build and publish + image: nixpkgs/nix:nixos-24.05 + environment: + DOCKER_PLATFORM: "linux/${ARCH}" + CONTAINER_NAME: "dxflrs/${ARCH}_garage" + DOCKER_AUTH: + from_secret: docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "to_docker" diff --git a/Cargo.lock b/Cargo.lock index 20385ffd..7473d9af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,54 +1,85 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" -version = "0.19.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "ahash" -version = "0.7.6" +name = "aead" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "getrandom", - "once_cell", - "version_check", + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", ] [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.3.4", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -59,16 +90,81 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.68" +name = "anstream" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] [[package]] name = "arrayvec" @@ -88,222 +184,215 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.15" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" dependencies = [ - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "zstd 0.11.2+zstd.1.5.2", - "zstd-safe 5.0.2+zstd.1.5.2", ] [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "async-trait" -version = "0.1.63" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] -name = "atty" -version = "0.2.14" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "aws-endpoint" -version = "0.49.0" +name = "aws-credential-types" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f35c8f5877ad60db4f0d9dcdfbcb2233a8cc539f9e568df39ee0581ec62e89" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" dependencies = [ - "aws-smithy-http", + "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", + "zeroize", ] [[package]] -name = "aws-http" -version = "0.49.0" +name = "aws-runtime" +version = "1.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f5422c9632d887968ccb66e2871a6d190d6104e276034912bee72ef58a5d890" +checksum = "959dab27ce613e6c9658eb3621064d0e2027e5f2acb65bc526a43577facea557" dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", - "http-body", - "lazy_static", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-config" +version = "1.99.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e62e5ffb669e13f084c4e1d89d687604e001187f61503606a7f8cc7a411995" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] name = "aws-sdk-s3" -version = "0.19.0" +version = "1.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d31765abb258c501d5572ebce43dee524b4b3b6256cb8b4c78534898dc205b" +checksum = "06673901e961f20fa8d7da907da48f7ad6c1b383e3726c22bd418900f015abe1" dependencies = [ - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-credential-types", + "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", - "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", - "bytes-utils", - "http", - "http-body", - "tokio-stream", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16f4d70c9c865af392eb40cacfe2bec3fa18f651fbdf49919cfc1dda13b189e" -dependencies = [ - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "lru", + "percent-encoding", + "regex-lite", + "sha2", "tracing", + "url", ] [[package]] name = "aws-sigv4" -version = "0.49.0" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d33790cecae42b999d197074c8a19e9b96b9e346284a6f93989e7489c9fa0f5" +checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", "form_urlencoded", "hex", - "http", - "once_cell", + "hmac", + "http 0.2.12", + "http 1.4.0", "percent-encoding", - "regex", - "ring", - "time 0.3.17", + "sha2", + "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "0.49.0" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc604f278bae64bbd15854baa9c46ed69a56dfb0669d04aab80974749f2d6599" +checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.49.0" +version = "0.63.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b402da39bc5aae618b70a9b8d828acad21fe4a3a73b82c0205b89db55d71ce8" +checksum = "23374b9170cbbcc6f5df8dc5ebb9b6c5c28a3c8f599f0e8b8b10eb6f4a5c6e74" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", - "crc32c", - "crc32fast", + "crc-fast", "hex", - "http", - "http-body", - "md-5 0.10.5", + "http 0.2.12", + "http-body 0.4.6", + "md-5", "pin-project-lite", "sha1", - "sha2 0.10.6", - "tracing", -] - -[[package]] -name = "aws-smithy-client" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec39585f8274fa543ad5c63cc09cbd435666be16b2cf99e4e07be5cf798bc050" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls 0.22.1", - "lazy_static", - "pin-project-lite", - "tokio", - "tower", + "sha2", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.49.0" +version = "0.60.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c2a7b9490fd2bc7af3a1c486ae921102d7234d1fa5e7d91039068e7af48a01" +checksum = "dc12f8b310e38cad85cf3bef45ad236f470717393c613266ce0a89512286b650" dependencies = [ "aws-smithy-types", "bytes", @@ -312,76 +401,154 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.49.0" +version = "0.62.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014a0ef5c4508fc2f6a9d3925c214725af19f020ea388db48e20196cc4cc9d6d" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", - "http", - "http-body", - "hyper", - "once_cell", + "futures-util", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", - "tokio", - "tokio-util 0.7.4", + "pin-utils", "tracing", ] [[package]] -name = "aws-smithy-http-tower" -version = "0.49.0" +name = "aws-smithy-http-client" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deecb478dc3cc40203e0e97ac0fb92947e0719754bbafd0026bdc49318e2fd03" +checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" dependencies = [ - "aws-smithy-http", - "bytes", - "http", - "http-body", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "pin-project-lite", - "tower", + "rustls 0.21.12", + "rustls-native-certs 0.8.3", + "tokio", "tracing", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1fcbefc7ece1d70dcce29e490f269695dfca2d2bacdeaf9e5c3f799e4e6a42" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5b6167fcdf47399024e81ac08e795180c576a20e4d4ce67949f9a88ae37dc1" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efce7aaaf59ad53c5412f14fc19b2d5c6ab2c3ec688d272fd31f76ec12f44fb0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + [[package]] name = "aws-smithy-types" -version = "0.49.0" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93b0c93a3b963da946a0b8ef3853a7252298eb75cdbfb21dad60f5ed0ded861" +checksum = "65f172bcb02424eb94425db8aed1b6d583b5104d4d5ddddf22402c661a320048" dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", - "time 0.3.17", + "serde", + "time", + "tokio", + "tokio-util 0.7.18", ] [[package]] name = "aws-smithy-xml" -version = "0.49.0" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b9efb4855b4acb29961a776d45680f3cbdd7c4783cbbae078da54c342575dd" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.49.0" +version = "1.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f3f349b39781849261db1c727369923bb97007cf7bd0deb3a6e9e461c8d38f" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" dependencies = [ + "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "http", "rustc_version", "tracing", - "zeroize", ] [[package]] @@ -390,24 +557,24 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "getrandom", + "getrandom 0.2.17", "instant", "rand", ] [[package]] name = "backtrace" -version = "0.3.67" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-link", ] [[package]] @@ -418,9 +585,31 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bincode" @@ -437,62 +626,59 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + [[package]] name = "blake2" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.10.6", + "digest", ] [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "bytemuck" -version = "1.13.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.3.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "bytes-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", "either", @@ -500,24 +686,33 @@ dependencies = [ [[package]] name = "bytesize" -version = "1.1.0" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" + +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" dependencies = [ + "find-msvc-tools", "jobserver", + "libc", + "shlex", ] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -526,19 +721,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] -name = "chrono" -version = "0.4.23" +name = "cfg_aliases" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", - "winapi", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", ] [[package]] @@ -547,63 +756,95 @@ version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "textwrap", "unicode-width", ] [[package]] name = "clap" -version = "4.1.4" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ - "bitflags", + "clap_builder", "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +dependencies = [ + "anstream", + "anstyle", "clap_lex", - "is-terminal", - "once_cell", "strsim", - "termcolor", ] [[package]] name = "clap_derive" -version = "4.1.0" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ - "heck 0.4.0", - "proc-macro-error", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" dependencies = [ - "os_str_bytes", + "compression-core", + "zstd", + "zstd-safe", ] [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "compression-core" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "termcolor", - "unicode-width", + "core-foundation-sys", + "libc", ] [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -611,157 +852,132 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] -name = "crc32c" -version = "0.6.3" +name = "crc" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest", + "rustversion", + "spin 0.10.0", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ - "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core", "typenum", ] [[package]] -name = "crypto-mac" -version = "0.11.1" +name = "ctr" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ct-logs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" -dependencies = [ - "sct 0.6.1", -] - -[[package]] -name = "cxx" -version = "1.0.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "cipher", ] [[package]] name = "darling" -version = "0.14.2" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -769,27 +985,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.2" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.114", ] [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.114", ] [[package]] @@ -802,6 +1018,29 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + [[package]] name = "derivative" version = "2.2.0" @@ -810,124 +1049,104 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "digest" -version = "0.9.0" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "generic-array", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer 0.10.3", + "block-buffer", "crypto-common", "subtle", ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "cfg-if", - "dirs-sys-next", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "double-ended-peekable" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" [[package]] name = "dyn-clone" -version = "1.0.10" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.8.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "env_logger" -version = "0.7.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "atty", "humantime", + "is-terminal", "log", "regex", "termcolor", ] [[package]] -name = "err-derive" -version = "0.3.1" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "rustversion", - "syn", - "synstructure", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.2.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.61.2", ] [[package]] name = "fallible-iterator" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fallible-streaming-iterator" @@ -937,12 +1156,15 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.8.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" -dependencies = [ - "instant", -] +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "fixedbitset" @@ -950,6 +1172,23 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fjall" +version = "2.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +dependencies = [ + "byteorder", + "byteview", + "dashmap 6.1.0", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "fnv" version = "1.0.7" @@ -957,29 +1196,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "form_urlencoded" -version = "1.1.0" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] [[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] +name = "format_table" +version = "0.1.1" [[package]] name = "futures" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -992,9 +1237,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1002,15 +1247,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1019,38 +1264,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1064,118 +1309,202 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "garage" -version = "0.8.2" +version = "1.3.1" dependencies = [ "assert-json-diff", "async-trait", "aws-sdk-s3", + "aws-smithy-runtime", "backtrace", - "base64 0.21.0", + "base64 0.21.7", "bytes", "bytesize", "chrono", + "crc32fast", + "format_table", "futures", - "futures-util", - "garage_api", + "garage_api_admin", + "garage_api_common", + "garage_api_k2v", + "garage_api_s3", "garage_block", "garage_db", "garage_model", + "garage_net", "garage_rpc", "garage_table", "garage_util", "garage_web", + "git-version", "hex", - "hmac 0.12.1", - "http", - "hyper", + "hmac", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "k2v-client", "kuska-sodiumoxide", - "netapp", + "mktemp", "opentelemetry", "opentelemetry-otlp", "opentelemetry-prometheus", "parse_duration", - "prometheus", - "rand", "serde", - "serde_bytes", "serde_json", - "sha2 0.10.6", + "sha1", + "sha2", "static_init", "structopt", + "syslog-tracing", "timeago", "tokio", - "toml", "tracing", + "tracing-journald", "tracing-subscriber", ] [[package]] -name = "garage_api" -version = "0.8.2" +name = "garage_api_admin" +version = "1.3.1" dependencies = [ + "argon2", "async-trait", - "base64 0.21.0", - "bytes", - "chrono", - "crypto-common", - "err-derive", - "form_urlencoded", "futures", - "futures-util", - "garage_block", + "garage_api_common", "garage_model", "garage_rpc", "garage_table", "garage_util", "hex", - "hmac 0.12.1", - "http", - "http-range", - "httpdate", - "hyper", - "idna", - "md-5 0.10.5", - "multer", - "nom", + "http 1.4.0", + "hyper 1.8.1", "opentelemetry", "opentelemetry-prometheus", + "prometheus", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_common" +version = "1.3.1" +dependencies = [ + "base64 0.21.7", + "bytes", + "chrono", + "crc32c", + "crc32fast", + "crypto-common", + "futures", + "garage_model", + "garage_table", + "garage_util", + "hex", + "hmac", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "md-5", + "nom", + "opentelemetry", + "pin-project", + "serde", + "serde_json", + "sha1", + "sha2", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_k2v" +version = "1.3.1" +dependencies = [ + "base64 0.21.7", + "futures", + "garage_api_common", + "garage_model", + "garage_table", + "garage_util", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "opentelemetry", + "percent-encoding", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "garage_api_s3" +version = "1.3.1" +dependencies = [ + "aes-gcm", + "async-compression", + "base64 0.21.7", + "bytes", + "chrono", + "crc32c", + "crc32fast", + "form_urlencoded", + "futures", + "garage_api_common", + "garage_block", + "garage_model", + "garage_net", + "garage_rpc", + "garage_table", + "garage_util", + "hex", + "http 1.4.0", + "http-body-util", + "http-range", + "httpdate", + "hyper 1.8.1", + "md-5", + "multer", + "opentelemetry", "percent-encoding", "pin-project", - "prometheus", "quick-xml", "roxmltree", "serde", - "serde_bytes", "serde_json", - "sha2 0.10.6", + "sha1", + "sha2", + "thiserror 2.0.18", "tokio", "tokio-stream", + "tokio-util 0.7.18", "tracing", "url", ] [[package]] name = "garage_block" -version = "0.8.2" +version = "1.3.1" dependencies = [ "arc-swap", "async-compression", "async-trait", "bytes", + "bytesize", "futures", - "futures-util", "garage_db", + "garage_net", "garage_rpc", "garage_table", "garage_util", @@ -1183,72 +1512,98 @@ dependencies = [ "opentelemetry", "rand", "serde", - "serde_bytes", "tokio", - "tokio-util 0.6.10", + "tokio-util 0.7.18", "tracing", - "zstd 0.12.2+zstd.1.5.2", + "zstd", ] [[package]] name = "garage_db" -version = "0.8.2" +version = "1.3.1" dependencies = [ - "clap 4.1.4", - "err-derive", + "fjall", "heed", - "hexdump", "mktemp", - "pretty_env_logger", + "parking_lot", + "r2d2", + "r2d2_sqlite", "rusqlite", - "sled", + "thiserror 2.0.18", "tracing", ] [[package]] name = "garage_model" -version = "0.8.2" +version = "1.3.1" dependencies = [ - "arc-swap", "async-trait", - "base64 0.21.0", + "base64 0.21.7", "blake2", - "err-derive", + "chrono", "futures", - "futures-util", "garage_block", "garage_db", + "garage_net", "garage_rpc", "garage_table", "garage_util", "hex", - "netapp", - "opentelemetry", + "http 1.4.0", + "parse_duration", "rand", "serde", "serde_bytes", + "thiserror 2.0.18", "tokio", "tracing", - "zstd 0.12.2+zstd.1.5.2", + "zstd", +] + +[[package]] +name = "garage_net" +version = "1.3.1" +dependencies = [ + "arc-swap", + "bytes", + "cfg-if", + "futures", + "hex", + "kuska-handshake", + "kuska-sodiumoxide", + "log", + "opentelemetry", + "opentelemetry-contrib", + "pin-project", + "pretty_env_logger", + "rand", + "rmp-serde", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util 0.7.18", ] [[package]] name = "garage_rpc" -version = "0.8.2" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", - "bytes", - "err-derive", + "bytesize", + "format_table", "futures", - "futures-util", + "garage_net", "garage_util", "gethostname", "hex", + "ipnet", + "itertools 0.12.1", "k8s-openapi", "kube", "kuska-sodiumoxide", - "netapp", + "nix", "opentelemetry", "pnet_datalink", "rand", @@ -1257,19 +1612,17 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "systemstat", + "thiserror 2.0.18", "tokio", - "tokio-stream", "tracing", ] [[package]] name = "garage_table" -version = "0.8.2" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", - "bytes", "futures", "futures-util", "garage_db", @@ -1287,32 +1640,30 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.8.2" +version = "1.3.1" dependencies = [ "arc-swap", "async-trait", "blake2", - "bytes", + "bytesize", "chrono", - "digest 0.10.6", - "err-derive", "futures", "garage_db", - "git-version", + "garage_net", "hex", "hexdump", - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "lazy_static", "mktemp", - "netapp", "opentelemetry", "rand", "rmp-serde", "rustc_version", "serde", "serde_json", - "sha2 0.10.6", + "sha2", + "thiserror 2.0.18", "tokio", "toml", "tracing", @@ -1321,26 +1672,28 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.8.2" +version = "1.3.1" dependencies = [ - "err-derive", - "futures", - "garage_api", + "garage_api_common", + "garage_api_s3", "garage_model", "garage_table", "garage_util", - "http", - "hyper", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", "opentelemetry", "percent-encoding", + "thiserror 2.0.18", + "tokio", "tracing", ] [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -1348,69 +1701,114 @@ dependencies = [ [[package]] name = "gethostname" -version = "0.2.3" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" dependencies = [ "libc", - "winapi", + "windows-targets 0.48.5", ] [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", ] [[package]] name = "gimli" -version = "0.27.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "git-version" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" dependencies = [ "git-version-macro", - "proc-macro-hack", ] [[package]] name = "git-version-macro" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ - "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] -name = "h2" -version = "0.3.15" +name = "guardian" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 0.2.12", + "indexmap 2.13.0", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.18", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util 0.7.18", "tracing", ] @@ -1419,17 +1817,44 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.7.6", + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] name = "hashlink" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown", + "hashbrown 0.15.5", ] [[package]] @@ -1443,9 +1868,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" @@ -1487,21 +1912,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1511,22 +1924,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hexdump" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2" +checksum = "cf31ab66ed8145a1c7427bd8e9b42a6131bd74ccf444f69b9e620c2e73ded832" dependencies = [ "arrayvec", - "itertools 0.4.19", -] - -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", ] [[package]] @@ -1535,14 +1937,23 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.6", + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", ] [[package]] name = "http" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1550,13 +1961,46 @@ dependencies = [ ] [[package]] -name = "http-body" -version = "0.4.5" +name = "http" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "http", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1568,49 +2012,46 @@ checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "1.3.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1618,35 +2059,60 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.22.1" +name = "hyper" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ - "ct-logs", - "futures-util", - "hyper", - "log", - "rustls 0.19.1", - "rustls-native-certs 0.5.0", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", "tokio", - "tokio-rustls 0.22.0", - "webpki 0.21.4", + "want", ] [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "http", - "hyper", + "futures-util", + "http 0.2.12", + "hyper 0.14.32", "log", - "rustls 0.20.8", - "rustls-native-certs 0.6.2", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.4.0", + "hyper 1.8.1", + "hyper-util", + "rustls 0.22.4", + "rustls-native-certs 0.7.3", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", ] [[package]] @@ -1655,34 +2121,142 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", ] [[package]] -name = "iana-time-zone" -version = "0.1.53" +name = "hyper-util" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.2", + "system-configuration 0.6.1", + "tokio", + "tower-layer", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "winapi", + "windows-core", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", ] [[package]] @@ -1693,48 +2267,77 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", ] [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] [[package]] -name = "io-lifetimes" -version = "1.0.4" +name = "interval-heap" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" dependencies = [ - "libc", - "windows-sys", + "compare", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "ipnetwork" @@ -1747,30 +2350,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", - "rustix", - "windows-sys", + "hermit-abi", + "libc", + "windows-sys 0.61.2", ] [[package]] -name = "isolang" -version = "2.2.0" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64fd6448ee8a45ce6e4365c58e4fa7d8740cba2ed70db3e9ab4879ebd93eaaa" -dependencies = [ - "phf", -] - -[[package]] -name = "itertools" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -1782,92 +2375,107 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.5" +name = "itertools" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "json-patch" -version = "0.2.7" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +checksum = "ec9ad60d674508f3ca8f380a928cfe7b096bc729c4e2dbfe3852bc45da3ab30b" dependencies = [ "serde", "serde_json", - "treediff", + "thiserror 1.0.69", ] [[package]] -name = "jsonpath_lib" -version = "0.3.0" +name = "jsonpath-rust" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +checksum = "96acbc6188d3bd83519d053efec756aa4419de62ec47be7f28dec297f7dc9eb0" dependencies = [ - "log", - "serde", + "pest", + "pest_derive", + "regex", "serde_json", + "thiserror 1.0.69", ] [[package]] name = "k2v-client" -version = "0.1.1" +version = "0.0.4" dependencies = [ - "base64 0.21.0", - "clap 4.1.4", - "garage_util", - "http", - "hyper-rustls 0.23.2", + "aws-sdk-config", + "aws-sigv4", + "base64 0.21.7", + "clap 4.5.54", + "format_table", + "hex", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.26.0", + "hyper-util", "log", - "rusoto_core", - "rusoto_credential", - "rusoto_signature", + "percent-encoding", "serde", "serde_json", - "thiserror", + "sha2", + "thiserror 2.0.18", "tokio", + "tracing-subscriber", ] [[package]] name = "k8s-openapi" -version = "0.16.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" +checksum = "550f99d93aa4c2b25de527bce492d772caf5e21d7ac9bd4b508ba781c8d91e30" dependencies = [ - "base64 0.13.1", - "bytes", + "base64 0.21.7", "chrono", - "http", - "percent-encoding", "serde", "serde-value", "serde_json", - "url", ] [[package]] name = "kube" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb19108692aeafebb108fd0a1c381c06ac4c03859652599420975165e939b8a" +checksum = "462fe330a0617b276ec864c2255810adcdf519ecb6844253c54074b2086a97bc" dependencies = [ "k8s-openapi", "kube-client", @@ -1878,35 +2486,35 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e1a80ecd1b1438a2fc004549e155d47250b9e01fbfcf4cfbe9c8b56a085593" +checksum = "7fe0d65dd6f3adba29cfb84f19dfe55449c7f6c35425f9d8294bec40313e0b64" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "bytes", "chrono", - "dirs-next", "either", "futures", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", + "home", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "hyper-timeout", - "jsonpath_lib", + "jsonpath-rust", "k8s-openapi", "kube-core", "pem", "pin-project", - "rustls 0.20.8", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "secrecy", "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.18", "tower", "tower-http", "tracing", @@ -1914,56 +2522,58 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4d780f2bb048eeef64a4c6b2582d26a0fe19e30b4d3cc9e081616e1779c5d47" +checksum = "a6b42844e9172f631b8263ea9ce003b9251da13beb1401580937ad206dd82f4c" dependencies = [ "chrono", "form_urlencoded", - "http", + "http 0.2.12", "json-patch", "k8s-openapi", "once_cell", "schemars", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "kube-derive" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98459d53b2841237392cd6959956185b2df15c19d32c3b275ed6ca7b7ee1adae" +checksum = "f5b5a111ee287bd237b8190b8c39543ea9fd22f79e9c32a36c24e08234bcda22" dependencies = [ "darling", "proc-macro2", "quote", "serde_json", - "syn", + "syn 2.0.114", ] [[package]] name = "kube-runtime" -version = "0.75.0" +version = "0.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7769af142ee2e46bfa44bd393cf7f40b9d8b80d2e11f6317399551ed17760beb" +checksum = "2bc06275064c81056fbb28ea876b3fb339d970e8132282119359afca0835c0ea" dependencies = [ - "ahash 0.8.3", + "ahash", + "async-trait", "backoff", "derivative", "futures", + "hashbrown 0.14.5", "json-patch", "k8s-openapi", "kube-client", - "parking_lot 0.12.1", + "parking_lot", "pin-project", "serde", "serde_json", "smallvec", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.18", "tracing", ] @@ -1977,7 +2587,7 @@ dependencies = [ "hex", "kuska-sodiumoxide", "log", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1993,15 +2603,15 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libsodium-sys" @@ -2017,9 +2627,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.25.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" dependencies = [ "cc", "pkg-config", @@ -2027,25 +2637,22 @@ dependencies = [ ] [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "linux-raw-sys" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lmdb-rkv-sys" @@ -2060,72 +2667,88 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.17" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" dependencies = [ - "cfg-if", + "hashbrown 0.16.1", ] [[package]] -name = "matchers" -version = "0.1.0" +name = "lsm-tree" +version = "2.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ "regex-automata", ] [[package]] name = "md-5" -version = "0.9.1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "md-5" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" -dependencies = [ - "digest 0.10.6", + "cfg-if", + "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -2135,49 +2758,47 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "wasi", + "windows-sys 0.61.2", ] [[package]] name = "mktemp" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a" +checksum = "69fed8fbcd01affec44ac226784c6476a6006d98d13e33bc0ca7977aaf046bd8" dependencies = [ "uuid", ] [[package]] name = "multer" -version = "2.0.4" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed4198ce7a4cbd2a57af78d28c6fbb57d81ac5f1d6ad79ac6c5587419cbdf22" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.4.0", "httparse", - "log", "memchr", "mime", - "spin 0.9.4", + "spin 0.9.8", "version_check", ] @@ -2188,30 +2809,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] -name = "netapp" -version = "0.5.2" +name = "nix" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "arc-swap", - "async-trait", - "bytes", + "bitflags 2.10.0", "cfg-if", - "err-derive", - "futures", - "hex", - "kuska-handshake", - "kuska-sodiumoxide", - "log", - "opentelemetry", - "opentelemetry-contrib", - "pin-project", - "rand", - "rmp-serde", - "serde", - "tokio", - "tokio-stream", - "tokio-util 0.7.4", + "cfg_aliases 0.2.1", + "libc", ] [[package]] @@ -2230,23 +2836,13 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "overload", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -2285,20 +2881,25 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num-conv" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -2319,49 +2920,61 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "object" -version = "0.30.3" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.17.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" @@ -2371,7 +2984,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" dependencies = [ "async-trait", "crossbeam-channel", - "dashmap", + "dashmap 4.0.2", "fnv", "futures-channel", "futures-executor", @@ -2381,7 +2994,7 @@ dependencies = [ "percent-encoding", "pin-project", "rand", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", ] @@ -2405,10 +3018,10 @@ dependencies = [ "async-trait", "futures", "futures-util", - "http", + "http 0.2.12", "opentelemetry", "prost", - "thiserror", + "thiserror 1.0.69", "tokio", "tonic", "tonic-build", @@ -2427,24 +3040,18 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] [[package]] -name = "os_str_bytes" -version = "6.4.1" +name = "outref" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "page_size" @@ -2458,50 +3065,25 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.6", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-link", ] [[package]] @@ -2516,79 +3098,128 @@ dependencies = [ ] [[package]] -name = "paste" -version = "1.0.11" +name = "password-hash" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde_core", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pest_meta" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" +dependencies = [ + "pest", + "sha2", +] [[package]] name = "petgraph" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap", -] - -[[package]] -name = "phf" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", + "indexmap 2.13.0", ] [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2598,24 +3229,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pnet_base" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e" +checksum = "fe4cf6fb3ab38b68d01ab2aea03ed3d1132b4868fa4e06285f29f16da01c5f4c" dependencies = [ "no-std-net", ] [[package]] name = "pnet_datalink" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce" +checksum = "ad5854abf0067ebbd3967f7d45ebc8976ff577ff0c7bd101c4973ae3c70f98fe" dependencies = [ "ipnetwork", "libc", @@ -2626,25 +3257,55 @@ dependencies = [ [[package]] name = "pnet_sys" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16" +checksum = "417c0becd1b573f6d544f73671070b039051e5ad819cc64aa96377b536128d00" dependencies = [ "libc", "winapi", ] [[package]] -name = "ppv-lite86" -version = "0.2.17" +name = "polyval" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_env_logger" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" dependencies = [ "env_logger", "log", @@ -2659,7 +3320,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -2674,34 +3335,28 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prometheus" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ "cfg-if", "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot", "protobuf", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -2744,7 +3399,7 @@ dependencies = [ "itertools 0.10.5", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2763,31 +3418,63 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quick-xml" -version = "0.21.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0452695941410a58c8ce4391707ba9bad26a247173bd9886a05a5e8a8babec75" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" dependencies = [ "memchr", "serde", ] [[package]] -name = "quote" -version = "1.0.23" +name = "quick_cache" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c" +dependencies = [ + "r2d2", + "rusqlite", + "uuid", +] + [[package]] name = "rand" version = "0.8.5" @@ -2815,34 +3502,35 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.17", ] [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall", - "thiserror", + "bitflags 2.10.0", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -2850,45 +3538,33 @@ dependencies = [ ] [[package]] -name = "regex-automata" -version = "0.1.10" +name = "regex-lite" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax", -] +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.0", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -2896,13 +3572,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.20.8", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", + "system-configuration 0.5.1", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -2913,126 +3591,50 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", + "cfg-if", + "getrandom 0.2.17", "libc", - "once_cell", - "spin 0.5.2", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.52.0", ] [[package]] name = "rmp" -version = "0.8.11" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" dependencies = [ - "byteorder", "num-traits", - "paste", ] [[package]] name = "rmp-serde" -version = "0.15.5" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" dependencies = [ - "byteorder", "rmp", "serde", ] [[package]] name = "roxmltree" -version = "0.14.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "rusoto_core" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" -dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes", - "crc32fast", - "futures", - "http", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version", - "serde", - "serde_json", - "tokio", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - -[[package]] -name = "rusoto_signature" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" -dependencies = [ - "base64 0.13.1", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http", - "hyper", - "log", - "md-5 0.9.1", - "percent-encoding", - "pin-project-lite", - "rusoto_credential", - "rustc_version", - "serde", - "sha2 0.9.9", - "tokio", -] +checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f" [[package]] name = "rusqlite" -version = "0.28.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" dependencies = [ - "bitflags", + "bitflags 2.10.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3042,102 +3644,172 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.36.7" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.10.0", "errno", - "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.19.1" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ - "base64 0.13.1", "log", "ring", - "sct 0.6.1", - "webpki 0.21.4", + "rustls-webpki 0.101.7", + "sct", ] [[package]] name = "rustls" -version = "0.20.8" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ - "log", "ring", - "sct 0.7.0", - "webpki 0.22.0", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.5.0" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ - "openssl-probe", - "rustls 0.19.1", + "openssl-probe 0.1.6", + "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] name = "rustls-native-certs" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ - "openssl-probe", - "rustls-pemfile", + "openssl-probe 0.1.6", + "rustls-pemfile 2.2.0", + "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.0", + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "same-file" @@ -3150,18 +3822,27 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", ] [[package]] name = "schemars" -version = "0.8.11" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5fb6c61f29e723026dc8e923d94c694313212abbecbbe5f55a7748eec5b307" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", @@ -3171,43 +3852,27 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.11" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f188d036977451159430f3b8dc82ec76364a42b7e289c2b18a9a18f4470058e9" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.114", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", @@ -3225,12 +3890,25 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", - "core-foundation", + "bitflags 2.10.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3238,26 +3916,33 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", ] [[package]] -name = "semver" -version = "1.0.16" +name = "self_cell" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -3273,52 +3958,63 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.8" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap", "itoa", - "ryu", + "memchr", "serde", + "serde_core", + "zmij", ] [[package]] name = "serde_spanned" -version = "0.6.0" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -3337,167 +4033,153 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.26" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.13.0", + "itoa", "ryu", "serde", - "yaml-rust", + "unsafe-libyaml", ] [[package]] name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.6", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "sha2" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.6", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", ] [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - [[package]] name = "slab" -version = "0.4.7" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" -dependencies = [ - "autocfg", -] - -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.4.7" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", - "winapi", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", ] [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "spin" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "static_init" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2a1c578e98c1c16fc3b8ec1328f7659a500737d7a0c6d625e73e830ff9c1f6" +checksum = "8bae1df58c5fea7502e8e352ec26b5579f6178e1fdb311e088580c980dee25ed" dependencies = [ - "bitflags", - "cfg_aliases", + "bitflags 1.3.2", + "cfg_aliases 0.2.1", "libc", - "parking_lot 0.11.2", - "parking_lot_core 0.8.6", + "parking_lot", + "parking_lot_core", "static_init_macro", "winapi", ] [[package]] name = "static_init_macro" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a2595fc3aa78f2d0e45dd425b22282dd863273761cc77780914b2cf3003acf" +checksum = "1389c88ddd739ec6d3f8f83343764a0e944cd23cfbf126a9796a714b0b6edd6f" dependencies = [ - "cfg_aliases", + "cfg_aliases 0.1.1", "memchr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] -name = "strsim" -version = "0.10.0" +name = "std-semaphore" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structopt" @@ -3520,26 +4202,43 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synchronoise" version = "1.0.1" @@ -3551,49 +4250,86 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", - "unicode-xid", + "syn 2.0.114", ] [[package]] -name = "systemstat" -version = "0.2.3" +name = "syslog-tracing" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2" +checksum = "d349bc2df408b4bf656709a29643641cef7f1795d708f88b105c626a8f64f6e4" dependencies = [ - "bytesize", - "lazy_static", "libc", - "nom", - "time 0.3.17", - "winapi", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] name = "tempfile" -version = "3.3.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ - "cfg-if", "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.3", + "windows-sys 0.61.2", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -3609,120 +4345,121 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "once_cell", + "cfg-if", ] [[package]] name = "time" -version = "0.1.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" -dependencies = [ - "serde", + "deranged", + "num-conv", + "powerfmt", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" dependencies = [ + "num-conv", "time-core", ] [[package]] name = "timeago" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5082dc942361cdfb74eab98bf995762d6015e5bb3a20bf7c5c71213778b4fcb4" -dependencies = [ - "chrono", - "isolang", -] +checksum = "a1710e589de0a76aaf295cd47a6699f6405737dbfd3cf2b75c92d000b548d0e6" [[package]] -name = "tinyvec" -version = "1.6.0" +name = "tinystr" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - [[package]] name = "tokio" -version = "1.25.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ - "autocfg", "bytes", "libc", - "memchr", "mio", - "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] name = "tokio-io-timeout" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" dependencies = [ "pin-project-lite", "tokio", @@ -3730,42 +4467,41 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "tokio-rustls" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.19.1", + "rustls 0.21.12", "tokio", - "webpki 0.21.4", ] [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls 0.20.8", + "rustls 0.22.4", + "rustls-pki-types", "tokio", - "webpki 0.22.0", ] [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -3788,9 +4524,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -3799,14 +4535,13 @@ dependencies = [ "pin-project-lite", "slab", "tokio", - "tracing", ] [[package]] name = "toml" -version = "0.6.0" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -3816,24 +4551,24 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.18.1" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap", - "nom8", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", + "winnow", ] [[package]] @@ -3848,10 +4583,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -3876,7 +4611,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3887,13 +4622,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.18", "tower-layer", "tower-service", "tracing", @@ -3901,18 +4636,19 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.5" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "base64 0.13.1", - "bitflags", + "base64 0.21.7", + "bitflags 2.10.0", "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "http-range-header", + "mime", "pin-project-lite", "tower-layer", "tower-service", @@ -3921,23 +4657,22 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3946,20 +4681,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.114", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -3976,26 +4711,37 @@ dependencies = [ ] [[package]] -name = "tracing-log" -version = "0.1.3" +name = "tracing-journald" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -4004,97 +4750,126 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "treediff" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" -dependencies = [ - "serde_json", -] - [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] -name = "unicode-bidi" -version = "0.3.10" +name = "ucd-trie" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] -name = "unicode-xid" -version = "0.2.4" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.3.1" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] -name = "uuid" -version = "1.2.2" +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ - "getrandom", + "getrandom 0.2.17", + "rand", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vcpkg" @@ -4104,85 +4879,82 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", - "winapi", "winapi-util", ] [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.83" +name = "wasm-bindgen" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4190,62 +4962,46 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn", - "wasm-bindgen-backend", + "syn 2.0.114", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.44", ] [[package]] @@ -4266,11 +5022,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -4280,149 +5036,482 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows-core" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "winreg" -version = "0.10.1" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ - "winapi", + "memchr", ] [[package]] -name = "xml-rs" -version = "0.8.4" +name = "winreg" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xmlparser" -version = "0.13.3" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.6" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "yoke" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "linked-hash-map", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", ] [[package]] name = "zeroize" -version = "1.5.7" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" +name = "zerotrie" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", + "displaydoc", + "yoke", + "zerofrom", ] [[package]] -name = "zstd" -version = "0.12.2+zstd.1.5.2" +name = "zerovec" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9262a83dc741c0b0ffec209881b45dbc232c21b02a2b9cb1adb93266e41303d" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ - "zstd-safe 6.0.2+zstd.1.5.2", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "6.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cf39f730b440bab43da8fb5faf5f254574462f73f260f85f7987f32154ff17" -dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.5+zstd.1.5.2" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.nix b/Cargo.nix deleted file mode 100644 index 14e5c260..00000000 --- a/Cargo.nix +++ /dev/null @@ -1,6276 +0,0 @@ -# This file was @generated by cargo2nix 0.11.0. -# It is not intended to be manually edited. - -args@{ - release ? true, - rootFeatures ? [ - "garage_db/default" - "garage_util/default" - "garage_rpc/default" - "garage_table/default" - "garage_block/default" - "garage_model/default" - "garage_api/default" - "garage_web/default" - "garage/default" - "k2v-client/default" - ], - rustPackages, - buildRustPackages, - hostPlatform, - hostPlatformCpu ? null, - hostPlatformFeatures ? [], - target ? null, - codegenOpts ? null, - profileOpts ? null, - rustcLinkFlags ? null, - rustcBuildFlags ? null, - mkRustCrate, - rustLib, - lib, - workspaceSrc, - ignoreLockHash, -}: -let - nixifiedLockHash = "af19dd561f2ce73d1f609040c1ebdf74420974d96959016da98d44b0e347f013"; - workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; - currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); - lockHashIgnored = if ignoreLockHash - then builtins.trace "Ignoring lock hash" ignoreLockHash - else ignoreLockHash; -in if !lockHashIgnored && (nixifiedLockHash != currentLockHash) then - throw ("Cargo.nix ${nixifiedLockHash} is out of sync with Cargo.lock ${currentLockHash}") -else let - inherit (rustLib) fetchCratesIo fetchCrateLocal fetchCrateGit fetchCrateAlternativeRegistry expandFeatures decideProfile genDrvsByProfile; - profilesByName = { - dev = builtins.fromTOML "lto = \"off\"\n"; - release = builtins.fromTOML "debug = true\n"; - }; - rootFeatures' = expandFeatures rootFeatures; - overridableMkRustCrate = f: - let - drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts rustcLinkFlags rustcBuildFlags; } // (f profileName))); - in { compileMode ? null, profileName ? decideProfile compileMode release }: - let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; }; -in -{ - cargo2nixVersion = "0.11.0"; - workspace = { - garage_db = rustPackages.unknown.garage_db."0.8.2"; - garage_util = rustPackages.unknown.garage_util."0.8.2"; - garage_rpc = rustPackages.unknown.garage_rpc."0.8.2"; - garage_table = rustPackages.unknown.garage_table."0.8.2"; - garage_block = rustPackages.unknown.garage_block."0.8.2"; - garage_model = rustPackages.unknown.garage_model."0.8.2"; - garage_api = rustPackages.unknown.garage_api."0.8.2"; - garage_web = rustPackages.unknown.garage_web."0.8.2"; - garage = rustPackages.unknown.garage."0.8.2"; - k2v-client = rustPackages.unknown.k2v-client."0.1.1"; - }; - "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.19.0" = overridableMkRustCrate (profileName: rec { - name = "addr2line"; - version = "0.19.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"; }; - dependencies = { - gimli = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gimli."0.27.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "adler"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec { - name = "ahash"; - version = "0.7.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") && (hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "windows" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "redox" || hostPlatform.parsed.kernel.name == "cloudabi" || hostPlatform.parsed.kernel.name == "haiku" || hostPlatform.parsed.kernel.name == "vxworks" || hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi") then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "ahash"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "getrandom") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "runtime-rng") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.20" = overridableMkRustCrate (profileName: rec { - name = "aho-corasick"; - version = "0.7.20"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".android_system_properties."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "android_system_properties"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.68" = overridableMkRustCrate (profileName: rec { - name = "anyhow"; - version = "1.0.68"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" = overridableMkRustCrate (profileName: rec { - name = "arc-swap"; - version = "1.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".arrayvec."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "arrayvec"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".assert-json-diff."2.0.2" = overridableMkRustCrate (profileName: rec { - name = "assert-json-diff"; - version = "2.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"; }; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-compression."0.3.15" = overridableMkRustCrate (profileName: rec { - name = "async-compression"; - version = "0.3.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "libzstd" ] - [ "tokio" ] - [ "zstd" ] - [ "zstd-safe" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - libzstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.11.2+zstd.1.5.2" { inherit profileName; }).out; - zstd_safe = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."5.0.2+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" = overridableMkRustCrate (profileName: rec { - name = "async-stream"; - version = "0.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_stream_impl" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" = overridableMkRustCrate (profileName: rec { - name = "async-stream-impl"; - version = "0.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" = overridableMkRustCrate (profileName: rec { - name = "async-trait"; - version = "0.1.63"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".atty."0.2.14" = overridableMkRustCrate (profileName: rec { - name = "atty"; - version = "0.2.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.1.19" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "autocfg"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-endpoint"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "76f35c8f5877ad60db4f0d9dcdfbcb2233a8cc539f9e568df39ee0581ec62e89"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.49.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-http"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2f5422c9632d887968ccb66e2871a6d190d6104e276034912bee72ef58a5d890"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.19.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sdk-s3"; - version = "0.19.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a4d31765abb258c501d5572ebce43dee524b4b3b6256cb8b4c78534898dc205b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "rt-tokio" ] - [ "rustls" ] - ]; - dependencies = { - aws_endpoint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.49.0" { inherit profileName; }).out; - aws_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.49.0" { inherit profileName; }).out; - aws_sig_auth = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.49.0" { inherit profileName; }).out; - aws_sigv4 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.49.0" { inherit profileName; }).out; - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.49.0" { inherit profileName; }).out; - aws_smithy_checksums = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-checksums."0.49.0" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.49.0" { inherit profileName; }).out; - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.49.0" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - aws_smithy_xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.49.0" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - bytes_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sig-auth"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b16f4d70c9c865af392eb40cacfe2bec3fa18f651fbdf49919cfc1dda13b189e"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "sign-eventstream" ] - ]; - dependencies = { - aws_sigv4 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.49.0" { inherit profileName; }).out; - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.49.0" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.49.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-sigv4"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8d33790cecae42b999d197074c8a19e9b96b9e346284a6f93989e7489c9fa0f5"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "bytes" ] - [ "default" ] - [ "form_urlencoded" ] - [ "http" ] - [ "percent-encoding" ] - [ "sign-eventstream" ] - [ "sign-http" ] - ]; - dependencies = { - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.49.0" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.17" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-async"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bc604f278bae64bbd15854baa9c46ed69a56dfb0669d04aab80974749f2d6599"; }; - features = builtins.concatLists [ - [ "rt-tokio" ] - ]; - dependencies = { - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-checksums."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-checksums"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4b402da39bc5aae618b70a9b8d828acad21fe4a3a73b82c0205b89db55d71ce8"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - crc32c = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32c."0.6.3" { inherit profileName; }).out; - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - sha1 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha1."0.10.5" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.6" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-client"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ec39585f8274fa543ad5c63cc09cbd435666be16b2cf99e4e07be5cf798bc050"; }; - features = builtins.concatLists [ - [ "client-hyper" ] - [ "hyper" ] - [ "hyper-rustls" ] - [ "lazy_static" ] - [ "rt-tokio" ] - [ "rustls" ] - ]; - dependencies = { - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.49.0" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_http_tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - fastrand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.8.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - hyper_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.22.1" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-eventstream"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "98c2a7b9490fd2bc7af3a1c486ae921102d7234d1fa5e7d91039068e7af48a01"; }; - dependencies = { - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-http"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "014a0ef5c4508fc2f6a9d3925c214725af19f020ea388db48e20196cc4cc9d6d"; }; - features = builtins.concatLists [ - [ "aws-smithy-eventstream" ] - [ "event-stream" ] - [ "rt-tokio" ] - [ "tokio" ] - [ "tokio-util" ] - ]; - dependencies = { - aws_smithy_eventstream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - bytes_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.3" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-http-tower"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "deecb478dc3cc40203e0e97ac0fb92947e0719754bbafd0026bdc49318e2fd03"; }; - dependencies = { - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tower = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-types"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e93b0c93a3b963da946a0b8ef3853a7252298eb75cdbfb21dad60f5ed0ded861"; }; - dependencies = { - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - ryu = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.12" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.17" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-smithy-xml"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "36b9efb4855b4acb29961a776d45680f3cbdd7c4783cbbae078da54c342575dd"; }; - dependencies = { - xmlparser = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.49.0" = overridableMkRustCrate (profileName: rec { - name = "aws-types"; - version = "0.49.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "93f3f349b39781849261db1c727369923bb97007cf7bd0deb3a6e9e461c8d38f"; }; - dependencies = { - aws_smithy_async = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.49.0" { inherit profileName; }).out; - aws_smithy_client = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.49.0" { inherit profileName; }).out; - aws_smithy_http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.49.0" { inherit profileName; }).out; - aws_smithy_types = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.49.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zeroize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.7" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".backoff."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "backoff"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "instant" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.67" = overridableMkRustCrate (profileName: rec { - name = "backtrace"; - version = "0.3.67"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - addr2line = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".addr2line."0.19.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - miniz_oxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.6.2" { inherit profileName; }).out; - object = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".object."0.30.3" { inherit profileName; }).out; - rustc_demangle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" = overridableMkRustCrate (profileName: rec { - name = "base64"; - version = "0.13.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" = overridableMkRustCrate (profileName: rec { - name = "base64"; - version = "0.21.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" = overridableMkRustCrate (profileName: rec { - name = "bincode"; - version = "1.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"; }; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" = overridableMkRustCrate (profileName: rec { - name = "bitflags"; - version = "1.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" = overridableMkRustCrate (profileName: rec { - name = "blake2"; - version = "0.10.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "block-buffer"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"; }; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.3" = overridableMkRustCrate (profileName: rec { - name = "block-buffer"; - version = "0.10.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"; }; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bumpalo."3.12.0" = overridableMkRustCrate (profileName: rec { - name = "bumpalo"; - version = "3.12.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.0" = overridableMkRustCrate (profileName: rec { - name = "bytemuck"; - version = "1.13.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "extern_crate_alloc") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "extern_crate_std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" = overridableMkRustCrate (profileName: rec { - name = "byteorder"; - version = "1.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" = overridableMkRustCrate (profileName: rec { - name = "bytes"; - version = "1.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.3" = overridableMkRustCrate (profileName: rec { - name = "bytes-utils"; - version = "0.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - either = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.8.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "bytesize"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" = overridableMkRustCrate (profileName: rec { - name = "cc"; - version = "1.0.79"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"; }; - features = builtins.concatLists [ - [ "jobserver" ] - [ "parallel" ] - ]; - dependencies = { - jobserver = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.25" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" = overridableMkRustCrate (profileName: rec { - name = "cfg-if"; - version = "1.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "cfg_aliases"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" = overridableMkRustCrate (profileName: rec { - name = "chrono"; - version = "0.4.23"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - [ "clock" ] - [ "default" ] - [ "iana-time-zone" ] - [ "js-sys" ] - [ "oldtime" ] - [ "serde" ] - [ "std" ] - [ "time" ] - [ "wasm-bindgen" ] - [ "wasmbind" ] - [ "winapi" ] - ]; - dependencies = { - iana_time_zone = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".iana-time-zone."0.1.53" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && !(hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi") then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.1.45" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && !(hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi") then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap."2.34.0" = overridableMkRustCrate (profileName: rec { - name = "clap"; - version = "2.34.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"; }; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - textwrap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".textwrap."0.11.0" { inherit profileName; }).out; - unicode_width = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap."4.1.4" = overridableMkRustCrate (profileName: rec { - name = "clap"; - version = "4.1.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "color") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "derive") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "env") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "error-context") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "help") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "std") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "suggestions") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "usage") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.1.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap_lex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.3.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "is_terminal" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".is-terminal."0.4.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "strsim" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "termcolor" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.1.0" = overridableMkRustCrate (profileName: rec { - name = "clap_derive"; - version = "4.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "heck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "proc_macro_error" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "clap_lex"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "os_str_bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".os_str_bytes."6.4.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".codespan-reporting."0.11.1" = overridableMkRustCrate (profileName: rec { - name = "codespan-reporting"; - version = "0.11.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"; }; - dependencies = { - termcolor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" { inherit profileName; }).out; - unicode_width = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".core-foundation."0.9.3" = overridableMkRustCrate (profileName: rec { - name = "core-foundation"; - version = "0.9.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"; }; - dependencies = { - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.3" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "core-foundation-sys"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.5" = overridableMkRustCrate (profileName: rec { - name = "cpufeatures"; - version = "0.2.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"; }; - dependencies = { - ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crc32c."0.6.3" = overridableMkRustCrate (profileName: rec { - name = "crc32c"; - version = "0.6.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e"; }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" = overridableMkRustCrate (profileName: rec { - name = "crc32fast"; - version = "1.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.6" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-channel"; - version = "0.5.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"; }; - features = builtins.concatLists [ - [ "crossbeam-utils" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.14" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.13" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-epoch"; - version = "0.9.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.14" { inherit profileName; }).out; - memoffset = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.7.1" { inherit profileName; }).out; - scopeguard = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.3.8" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-queue"; - version = "0.3.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "alloc") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "default") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.14" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.14" = overridableMkRustCrate (profileName: rec { - name = "crossbeam-utils"; - version = "0.8.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" = overridableMkRustCrate (profileName: rec { - name = "crypto-common"; - version = "0.1.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" { inherit profileName; }).out; - typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".crypto-mac."0.11.1" = overridableMkRustCrate (profileName: rec { - name = "crypto-mac"; - version = "0.11.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"; }; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" { inherit profileName; }).out; - subtle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ct-logs."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "ct-logs"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"; }; - dependencies = { - sct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sct."0.6.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cxx."1.0.88" = overridableMkRustCrate (profileName: rec { - name = "cxx"; - version = "1.0.88"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - cxxbridge_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cxxbridge-macro."1.0.88" { profileName = "__noProfile"; }).out; - link_cplusplus = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".link-cplusplus."1.0.8" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - cxxbridge_flags = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cxxbridge-flags."1.0.88" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cxx-build."1.0.88" = overridableMkRustCrate (profileName: rec { - name = "cxx-build"; - version = "1.0.88"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8"; }; - dependencies = { - cc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { inherit profileName; }).out; - codespan_reporting = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".codespan-reporting."0.11.1" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - scratch = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scratch."1.0.3" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cxxbridge-flags."1.0.88" = overridableMkRustCrate (profileName: rec { - name = "cxxbridge-flags"; - version = "1.0.88"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".cxxbridge-macro."1.0.88" = overridableMkRustCrate (profileName: rec { - name = "cxxbridge-macro"; - version = "1.0.88"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling."0.14.2" = overridableMkRustCrate (profileName: rec { - name = "darling"; - version = "0.14.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "suggestions") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_macro" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_macro."0.14.2" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.2" = overridableMkRustCrate (profileName: rec { - name = "darling_core"; - version = "0.14.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "strsim") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "suggestions") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "fnv" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ident_case" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ident_case."1.0.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "strsim" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".darling_macro."0.14.2" = overridableMkRustCrate (profileName: rec { - name = "darling_macro"; - version = "0.14.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling_core."0.14.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" = overridableMkRustCrate (profileName: rec { - name = "dashmap"; - version = "4.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - num_cpus = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.15.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".derivative."2.2.0" = overridableMkRustCrate (profileName: rec { - name = "derivative"; - version = "2.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "digest"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - dependencies = { - generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" = overridableMkRustCrate (profileName: rec { - name = "digest"; - version = "0.10.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "block-buffer" ] - [ "core-api" ] - [ "default" ] - [ "mac" ] - [ "std" ] - [ "subtle" ] - ]; - dependencies = { - block_buffer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.3" { inherit profileName; }).out; - crypto_common = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; }).out; - subtle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" = overridableMkRustCrate (profileName: rec { - name = "dirs-next"; - version = "2.0.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - dirs_sys_next = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-sys-next."0.1.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dirs-sys-next."0.1.2" = overridableMkRustCrate (profileName: rec { - name = "dirs-sys-next"; - version = "0.1.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"; }; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "redox" then "redox_users" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_users."0.4.3" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".dyn-clone."1.0.10" = overridableMkRustCrate (profileName: rec { - name = "dyn-clone"; - version = "1.0.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".either."1.8.1" = overridableMkRustCrate (profileName: rec { - name = "either"; - version = "1.8.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"; }; - features = builtins.concatLists [ - [ "default" ] - [ "use_std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.31" = overridableMkRustCrate (profileName: rec { - name = "encoding_rs"; - version = "0.8.31"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".env_logger."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "env_logger"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "atty") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "default") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "humantime") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "regex") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "termcolor") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "atty" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".atty."0.2.14" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "humantime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".humantime."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "regex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "termcolor" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "err-derive"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - proc_macro_error = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - synstructure = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".synstructure."0.12.6" { inherit profileName; }).out; - }; - buildDependencies = { - rustversion = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.11" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".errno."0.2.8" = overridableMkRustCrate (profileName: rec { - name = "errno"; - version = "0.2.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.parsed.kernel.name == "dragonfly" then "errno_dragonfly" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".errno-dragonfly."0.1.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && (hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "hermit" || hostPlatform.parsed.kernel.name == "wasi") then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".errno-dragonfly."0.1.2" = overridableMkRustCrate (profileName: rec { - name = "errno-dragonfly"; - version = "0.1.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "fallible-iterator"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" = overridableMkRustCrate (profileName: rec { - name = "fallible-streaming-iterator"; - version = "0.1.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fastrand."1.8.0" = overridableMkRustCrate (profileName: rec { - name = "fastrand"; - version = "1.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499"; }; - dependencies = { - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "instant" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.2" = overridableMkRustCrate (profileName: rec { - name = "fixedbitset"; - version = "0.4.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" = overridableMkRustCrate (profileName: rec { - name = "fnv"; - version = "1.0.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "form_urlencoded"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"; }; - dependencies = { - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "fs2"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"; }; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "async-await" ] - [ "default" ] - [ "executor" ] - [ "futures-executor" ] - [ "std" ] - ]; - dependencies = { - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.25" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_executor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.25" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.25" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-channel"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "futures-sink" ] - [ "sink" ] - [ "std" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-core"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-executor"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-io"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-macro"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-sink"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-task"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" = overridableMkRustCrate (profileName: rec { - name = "futures-util"; - version = "0.3.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "async-await" ] - [ "async-await-macro" ] - [ "channel" ] - [ "default" ] - [ "futures-channel" ] - [ "futures-io" ] - [ "futures-macro" ] - [ "futures-sink" ] - [ "io" ] - [ "memchr" ] - [ "sink" ] - [ "slab" ] - [ "std" ] - ]; - dependencies = { - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.25" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.25" { inherit profileName; }).out; - futures_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.25" { profileName = "__noProfile"; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - futures_task = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.25" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - pin_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" { inherit profileName; }).out; - slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "fxhash"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"; }; - dependencies = { - byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - }; - }); - - "unknown".garage."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/garage"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs") - (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery") - (lib.optional (rootFeatures' ? "garage/default") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery") - (lib.optional (rootFeatures' ? "garage/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "opentelemetry-otlp") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus") "opentelemetry-prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus") "prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled") "sled") - (lib.optional (rootFeatures' ? "garage/sqlite") "sqlite") - (lib.optional (rootFeatures' ? "garage/system-libs") "system-libs") - (lib.optional (rootFeatures' ? "garage/telemetry-otlp") "telemetry-otlp") - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - backtrace = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.67" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.8.2" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.2" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - garage_web = (rustPackages."unknown".garage_web."0.8.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out; - parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out; - timeago = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.1" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - tracing_subscriber = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.16" { inherit profileName; }).out; - }; - devDependencies = { - assert_json_diff = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".assert-json-diff."2.0.2" { inherit profileName; }).out; - aws_sdk_s3 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.19.0" { inherit profileName; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.6" { inherit profileName; }).out; - static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out; - }; - }); - - "unknown".garage_api."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_api"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/api"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus") - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - crypto_common = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_range = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-range."0.1.5" { inherit profileName; }).out; - httpdate = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - idna = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.3.0" { inherit profileName; }).out; - md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" { inherit profileName; }).out; - multer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.0.4" { inherit profileName; }).out; - nom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - quick_xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; }).out; - roxmltree = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.6" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - url = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.3.1" { inherit profileName; }).out; - }; - }); - - "unknown".garage_block."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_block"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/block"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "system-libs") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_compression = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-compression."0.3.15" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.10" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.2+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "unknown".garage_db."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_db"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/db"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli") "clap") - (lib.optional (rootFeatures' ? "garage_db/cli") "cli") - [ "default" ] - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "heed") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "pretty_env_logger") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "rusqlite") - [ "sled" ] - (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.1.4" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "heed" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "pretty_env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.28.0" { inherit profileName; }).out; - sled = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - devDependencies = { - mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out; - }; - }); - - "unknown".garage_model."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_model"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/model"); - features = builtins.concatLists [ - [ "default" ] - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb") - [ "sled" ] - (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.8.2" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - zstd = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.2+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "unknown".garage_rpc."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_rpc"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/rpc"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery") "consul-discovery") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive") "err-derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "k8s-openapi") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kubernetes-discovery") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "reqwest") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "schemars") - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "system-libs") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.75.0" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.14" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - systemstat = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "unknown".garage_table."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_table"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/table"); - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "unknown".garage_util."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_util"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/util"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v") - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.8.2" { inherit profileName; }).out; - git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.6" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - xxhash_rust = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.6" { inherit profileName; }).out; - }; - devDependencies = { - mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "unknown".garage_web."0.8.2" = overridableMkRustCrate (profileName: rec { - name = "garage_web"; - version = "0.8.2"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/web"); - dependencies = { - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.8.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.8.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.8.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.6" = overridableMkRustCrate (profileName: rec { - name = "generic-array"; - version = "0.14.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"; }; - features = builtins.concatLists [ - [ "more_lengths" ] - ]; - dependencies = { - typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" = overridableMkRustCrate (profileName: rec { - name = "gethostname"; - version = "0.2.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e"; }; - dependencies = { - ${ if !hostPlatform.isWindows then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" = overridableMkRustCrate (profileName: rec { - name = "getrandom"; - version = "0.2.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".gimli."0.27.1" = overridableMkRustCrate (profileName: rec { - name = "gimli"; - version = "0.27.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec"; }; - features = builtins.concatLists [ - [ "read" ] - [ "read-core" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "git-version"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899"; }; - dependencies = { - git_version_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version-macro."0.3.5" { profileName = "__noProfile"; }).out; - proc_macro_hack = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".git-version-macro."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "git-version-macro"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f"; }; - dependencies = { - proc_macro_hack = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" { profileName = "__noProfile"; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".h2."0.3.15" = overridableMkRustCrate (profileName: rec { - name = "h2"; - version = "0.3.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - indexmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.12.3" = overridableMkRustCrate (profileName: rec { - name = "hashbrown"; - version = "0.12.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "ahash") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "inline-more") - [ "raw" ] - ]; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.1" = overridableMkRustCrate (profileName: rec { - name = "hashlink"; - version = "0.8.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"; }; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.12.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" = overridableMkRustCrate (profileName: rec { - name = "heck"; - version = "0.3.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"; }; - dependencies = { - unicode_segmentation = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-segmentation."1.10.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heck."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "heck"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" = overridableMkRustCrate (profileName: rec { - name = "heed"; - version = "0.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb-rkv-sys") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "bytemuck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "heed_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "heed_types" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "lmdb_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "page_size" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "synchronoise" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isWindows then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.3.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "heed-traits"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "heed-types"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "bincode") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "default") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "serde") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "serde-bincode") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "serde-json") - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "serde_json") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "bincode" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "bytemuck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.13.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "heed_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.1.19" = overridableMkRustCrate (profileName: rec { - name = "hermit-abi"; - version = "0.1.19"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.2.6" = overridableMkRustCrate (profileName: rec { - name = "hermit-abi"; - version = "0.2.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "hex"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "hexdump"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2"; }; - dependencies = { - arrayvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arrayvec."0.5.2" { inherit profileName; }).out; - itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.4.19" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hmac."0.11.0" = overridableMkRustCrate (profileName: rec { - name = "hmac"; - version = "0.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"; }; - dependencies = { - crypto_mac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-mac."0.11.1" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" = overridableMkRustCrate (profileName: rec { - name = "hmac"; - version = "0.12.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"; }; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" = overridableMkRustCrate (profileName: rec { - name = "http"; - version = "0.2.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" = overridableMkRustCrate (profileName: rec { - name = "http-body"; - version = "0.4.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"; }; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-range."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "http-range"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".http-range-header."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "http-range-header"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" = overridableMkRustCrate (profileName: rec { - name = "httparse"; - version = "1.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "httpdate"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".humantime."1.3.0" = overridableMkRustCrate (profileName: rec { - name = "humantime"; - version = "1.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "quick_error" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-error."1.2.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" = overridableMkRustCrate (profileName: rec { - name = "hyper"; - version = "0.14.23"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"; }; - features = builtins.concatLists [ - [ "client" ] - [ "default" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "full") - [ "h2" ] - [ "http1" ] - [ "http2" ] - [ "runtime" ] - [ "server" ] - [ "socket2" ] - [ "stream" ] - [ "tcp" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.25" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - h2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.15" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - http_body = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - httparse = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" { inherit profileName; }).out; - httpdate = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - socket2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tower_service = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - want = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".want."0.3.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.22.1" = overridableMkRustCrate (profileName: rec { - name = "hyper-rustls"; - version = "0.22.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"; }; - features = builtins.concatLists [ - [ "ct-logs" ] - [ "default" ] - [ "native-tokio" ] - [ "rustls-native-certs" ] - [ "tokio-runtime" ] - ]; - dependencies = { - ct_logs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ct-logs."0.8.0" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.19.1" { inherit profileName; }).out; - rustls_native_certs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.5.0" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.22.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.21.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" = overridableMkRustCrate (profileName: rec { - name = "hyper-rustls"; - version = "0.23.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - [ "http1" ] - [ "http2" ] - [ "log" ] - [ "logging" ] - [ "native-tokio" ] - [ "rustls-native-certs" ] - [ "tls12" ] - [ "tokio-runtime" ] - ]; - dependencies = { - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - rustls_native_certs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.6.2" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.23.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "hyper-timeout"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_io_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-io-timeout."1.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".iana-time-zone."0.1.53" = overridableMkRustCrate (profileName: rec { - name = "iana-time-zone"; - version = "0.1.53"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"; }; - features = builtins.concatLists [ - [ "fallback" ] - ]; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "android" then "android_system_properties" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".android_system_properties."0.1.5" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios" then "core_foundation_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.3" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "haiku" then "iana_time_zone_haiku" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".iana-time-zone-haiku."0.1.1" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".iana-time-zone-haiku."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "iana-time-zone-haiku"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"; }; - dependencies = { - cxx = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cxx."1.0.88" { inherit profileName; }).out; - }; - buildDependencies = { - cxx_build = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cxx-build."1.0.88" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ident_case."1.0.1" = overridableMkRustCrate (profileName: rec { - name = "ident_case"; - version = "1.0.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".idna."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "idna"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"; }; - dependencies = { - unicode_bidi = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-bidi."0.3.10" { inherit profileName; }).out; - unicode_normalization = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-normalization."0.1.22" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" = overridableMkRustCrate (profileName: rec { - name = "indexmap"; - version = "1.9.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - hashbrown = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.12.3" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" = overridableMkRustCrate (profileName: rec { - name = "instant"; - version = "0.1.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".io-lifetimes."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "io-lifetimes"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "close") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "libc") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "windows-sys") - ]; - dependencies = { - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && !hostPlatform.isWindows then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ipnet."2.7.1" = overridableMkRustCrate (profileName: rec { - name = "ipnet"; - version = "2.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" = overridableMkRustCrate (profileName: rec { - name = "ipnetwork"; - version = "0.20.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".is-terminal."0.4.2" = overridableMkRustCrate (profileName: rec { - name = "is-terminal"; - version = "0.4.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189"; }; - dependencies = { - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.2.6" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "io_lifetimes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".io-lifetimes."1.0.4" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && !(hostPlatform.isWindows || hostPlatform.parsed.kernel.name == "hermit" || hostPlatform.parsed.kernel.name == "unknown") then "rustix" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."0.36.7" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".isolang."2.2.0" = overridableMkRustCrate (profileName: rec { - name = "isolang"; - version = "2.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b64fd6448ee8a45ce6e4365c58e4fa7d8740cba2ed70db3e9ab4879ebd93eaaa"; }; - features = builtins.concatLists [ - [ "default" ] - [ "english_names" ] - ]; - dependencies = { - phf = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf."0.10.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itertools."0.4.19" = overridableMkRustCrate (profileName: rec { - name = "itertools"; - version = "0.4.19"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "itertools"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "use_alloc") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "use_std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "either" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.8.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" = overridableMkRustCrate (profileName: rec { - name = "itoa"; - version = "1.0.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.25" = overridableMkRustCrate (profileName: rec { - name = "jobserver"; - version = "0.1.25"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"; }; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" = overridableMkRustCrate (profileName: rec { - name = "js-sys"; - version = "0.3.60"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"; }; - dependencies = { - wasm_bindgen = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" = overridableMkRustCrate (profileName: rec { - name = "json-patch"; - version = "0.2.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "diff") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "treediff") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "treediff" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".jsonpath_lib."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "jsonpath_lib"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - }; - }); - - "unknown".k2v-client."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "k2v-client"; - version = "0.1.1"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/k2v-client"); - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "clap") - (lib.optional (rootFeatures' ? "k2v-client/cli") "cli") - (lib.optional (rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util") "garage_util") - ]; - dependencies = { - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.1.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.2" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out; - rusoto_credential = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" { inherit profileName; }).out; - rusoto_signature = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_signature."0.48.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" = overridableMkRustCrate (profileName: rec { - name = "k8s-openapi"; - version = "0.16.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "api") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "http") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "percent-encoding") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "url") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "v1_22") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_value" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde-value."0.7.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.3.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9bb19108692aeafebb108fd0a1c381c06ac4c03859652599420975165e939b8a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "config") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "kube-runtime") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "runtime") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-tls") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_client" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-derive."0.75.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_runtime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-runtime."0.75.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-client"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "97e1a80ecd1b1438a2fc004549e155d47250b9e01fbfcf4cfbe9c8b56a085593"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "__non_core") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "base64") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "bytes") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "chrono") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "client") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "config") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "dirs") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "either") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "futures") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "http-body") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper-rustls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "hyper-timeout") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpatch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpath_lib") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "pem") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "pin-project") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-pemfile") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "rustls-tls") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde_yaml") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio-util") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tower") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tower-http") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tracing") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "dirs" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "either" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.8.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "hyper_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "jsonpath_lib" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".jsonpath_lib."0.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pem" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pem."1.1.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "rustls_pemfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "secrecy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".secrecy."0.8.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_yaml" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_yaml."0.8.26" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-http."0.3.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-core."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-core"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f4d780f2bb048eeef64a4c6b2582d26a0fe19e30b4d3cc9e081616e1779c5d47"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "json-patch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "jsonpatch") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "schema") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "schemars") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "chrono" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "form_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "json_patch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-derive."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-derive"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "98459d53b2841237392cd6959956185b2df15c19d32c3b275ed6ca7b7ee1adae"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "darling" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".darling."0.14.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kube-runtime."0.75.0" = overridableMkRustCrate (profileName: rec { - name = "kube-runtime"; - version = "0.75.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7769af142ee2e46bfa44bd393cf7f40b9d8b80d2e11f6317399551ed17760beb"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "backoff" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backoff."0.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "derivative" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".derivative."2.2.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "json_patch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".json-patch."0.2.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube_client" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube-client."0.75.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "kuska-handshake"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e33da4b69f23c2ece0b3e729d079cebdc2c0206e493e42f510f500ad81c631d5"; }; - features = builtins.concatLists [ - [ "async_std" ] - [ "default" ] - [ "futures" ] - ]; - dependencies = { - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" = overridableMkRustCrate (profileName: rec { - name = "kuska-sodiumoxide"; - version = "0.2.5-0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ae0f8eafdd240b722243787b51fdaf8df6693fb8621d0f7061cdba574214cf88"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - [ "std" ] - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "use-pkg-config") - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - libsodium_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsodium-sys."0.2.7" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" = overridableMkRustCrate (profileName: rec { - name = "lazy_static"; - version = "1.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" = overridableMkRustCrate (profileName: rec { - name = "libc"; - version = "0.2.139"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"; }; - features = builtins.concatLists [ - [ "default" ] - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "extra_traits") - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libsodium-sys."0.2.7" = overridableMkRustCrate (profileName: rec { - name = "libsodium-sys"; - version = "0.2.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6b779387cd56adfbc02ea4a668e704f729be8d6a6abd2c27ca5ee537849a92fd"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_rpc/system-libs") "use-pkg-config") - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if !(hostPlatform.parsed.abi.name == "msvc") then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.parsed.abi.name == "msvc" then "libc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { profileName = "__noProfile"; }).out; - pkg_config = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.26" { profileName = "__noProfile"; }).out; - walkdir = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".walkdir."2.3.2" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.25.2" = overridableMkRustCrate (profileName: rec { - name = "libsqlite3-sys"; - version = "0.25.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled_bindings") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "cc") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "default") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_6_8") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "pkg-config") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "vcpkg") - ]; - buildDependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.26" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".link-cplusplus."1.0.8" = overridableMkRustCrate (profileName: rec { - name = "link-cplusplus"; - version = "1.0.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".linked-hash-map."0.5.6" = overridableMkRustCrate (profileName: rec { - name = "linked-hash-map"; - version = "0.5.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.1.4" = overridableMkRustCrate (profileName: rec { - name = "linux-raw-sys"; - version = "0.1.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "errno") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "general") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "ioctl") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "no_std") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" = overridableMkRustCrate (profileName: rec { - name = "lmdb-rkv-sys"; - version = "0.11.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.26" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.9" = overridableMkRustCrate (profileName: rec { - name = "lock_api"; - version = "0.4.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"; }; - dependencies = { - scopeguard = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" = overridableMkRustCrate (profileName: rec { - name = "log"; - version = "0.4.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".matchers."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "matchers"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"; }; - dependencies = { - regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" = overridableMkRustCrate (profileName: rec { - name = "md-5"; - version = "0.9.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - block_buffer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.9.0" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" { inherit profileName; }).out; - opaque_debug = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opaque-debug."0.3.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "md-5"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" = overridableMkRustCrate (profileName: rec { - name = "memchr"; - version = "2.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".memoffset."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "memoffset"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mime."0.3.16" = overridableMkRustCrate (profileName: rec { - name = "mime"; - version = "0.3.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "minimal-lexical"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "miniz_oxide"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"; }; - dependencies = { - adler = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mio."0.8.5" = overridableMkRustCrate (profileName: rec { - name = "mio"; - version = "0.8.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"; }; - features = builtins.concatLists [ - [ "default" ] - [ "net" ] - [ "os-ext" ] - [ "os-poll" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "mktemp"; - version = "0.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a"; }; - dependencies = { - uuid = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".uuid."1.2.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".multer."2.0.4" = overridableMkRustCrate (profileName: rec { - name = "multer"; - version = "2.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6ed4198ce7a4cbd2a57af78d28c6fbb57d81ac5f1d6ad79ac6c5587419cbdf22"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - encoding_rs = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.31" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - httparse = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.8.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - mime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mime."0.3.16" { inherit profileName; }).out; - spin = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.9.4" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" = overridableMkRustCrate (profileName: rec { - name = "multimap"; - version = "0.8.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "netapp"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"; }; - features = builtins.concatLists [ - [ "default" ] - [ "opentelemetry" ] - [ "opentelemetry-contrib" ] - [ "telemetry" ] - ]; - dependencies = { - arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - kuska_handshake = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }).out; - sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - opentelemetry_contrib = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - tokio_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" = overridableMkRustCrate (profileName: rec { - name = "no-std-net"; - version = "0.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" = overridableMkRustCrate (profileName: rec { - name = "nom"; - version = "7.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - minimal_lexical = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nom8."0.2.0" = overridableMkRustCrate (profileName: rec { - name = "nom8"; - version = "0.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "std" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".nu-ansi-term."0.46.0" = overridableMkRustCrate (profileName: rec { - name = "nu-ansi-term"; - version = "0.46.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"; }; - dependencies = { - overload = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".overload."0.1.1" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" = overridableMkRustCrate (profileName: rec { - name = "num"; - version = "0.2.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"; }; - features = builtins.concatLists [ - [ "default" ] - [ "num-bigint" ] - [ "std" ] - ]; - dependencies = { - num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; - num_complex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_iter = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" { inherit profileName; }).out; - num_rational = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" = overridableMkRustCrate (profileName: rec { - name = "num-bigint"; - version = "0.2.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "num-complex"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" = overridableMkRustCrate (profileName: rec { - name = "num-integer"; - version = "0.1.45"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" = overridableMkRustCrate (profileName: rec { - name = "num-iter"; - version = "0.1.43"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "num-rational"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"; }; - features = builtins.concatLists [ - [ "bigint" ] - [ "num-bigint" ] - [ "std" ] - ]; - dependencies = { - num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; - num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.45" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" = overridableMkRustCrate (profileName: rec { - name = "num-traits"; - version = "0.2.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.15.0" = overridableMkRustCrate (profileName: rec { - name = "num_cpus"; - version = "1.15.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"; }; - dependencies = { - ${ if (hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "aarch64") && hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.2.6" { inherit profileName; }).out; - ${ if !hostPlatform.isWindows then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".object."0.30.3" = overridableMkRustCrate (profileName: rec { - name = "object"; - version = "0.30.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"; }; - features = builtins.concatLists [ - [ "archive" ] - [ "coff" ] - [ "elf" ] - [ "macho" ] - [ "pe" ] - [ "read_core" ] - [ "unaligned" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" = overridableMkRustCrate (profileName: rec { - name = "once_cell"; - version = "1.17.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "race" ] - [ "std" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "unstable") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opaque-debug."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "opaque-debug"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".openssl-probe."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "openssl-probe"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry"; - version = "0.17.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"; }; - features = builtins.concatLists [ - [ "async-trait" ] - [ "crossbeam-channel" ] - [ "dashmap" ] - [ "default" ] - [ "fnv" ] - [ "metrics" ] - [ "percent-encoding" ] - [ "pin-project" ] - [ "rand" ] - [ "rt-tokio" ] - [ "tokio" ] - [ "tokio-stream" ] - [ "trace" ] - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - crossbeam_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.6" { inherit profileName; }).out; - dashmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; }).out; - fnv = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - futures_channel = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.25" { inherit profileName; }).out; - futures_executor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-contrib"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85637add8f60bb4cac673469c14f47a329c6cec7365c72d72cd32f2d104a721a"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-otlp"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tokio") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tonic") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tonic-build") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_trait" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tonic" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tonic_build" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "opentelemetry-prometheus"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "opentelemetry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" then "protobuf" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ordered-float."2.10.0" = overridableMkRustCrate (profileName: rec { - name = "ordered-float"; - version = "2.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "num_traits" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".os_str_bytes."6.4.1" = overridableMkRustCrate (profileName: rec { - name = "os_str_bytes"; - version = "6.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "raw_os_str") - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".overload."0.1.1" = overridableMkRustCrate (profileName: rec { - name = "overload"; - version = "0.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" = overridableMkRustCrate (profileName: rec { - name = "page_size"; - version = "0.4.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" = overridableMkRustCrate (profileName: rec { - name = "parking_lot"; - version = "0.11.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - instant = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - lock_api = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.9" { inherit profileName; }).out; - parking_lot_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" = overridableMkRustCrate (profileName: rec { - name = "parking_lot"; - version = "0.12.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" = overridableMkRustCrate (profileName: rec { - name = "parking_lot_core"; - version = "0.8.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"; }; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - instant = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".instant."0.1.12" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - smallvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.6" = overridableMkRustCrate (profileName: rec { - name = "parking_lot_core"; - version = "0.9.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"; }; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" = overridableMkRustCrate (profileName: rec { - name = "parse_duration"; - version = "2.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"; }; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - num = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".paste."1.0.11" = overridableMkRustCrate (profileName: rec { - name = "paste"; - version = "1.0.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pem."1.1.1" = overridableMkRustCrate (profileName: rec { - name = "pem"; - version = "1.1.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" = overridableMkRustCrate (profileName: rec { - name = "percent-encoding"; - version = "2.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "petgraph"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "fixedbitset" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".phf."0.10.1" = overridableMkRustCrate (profileName: rec { - name = "phf"; - version = "0.10.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - phf_shared = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.10.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "phf_shared"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - siphasher = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".siphasher."0.3.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" = overridableMkRustCrate (profileName: rec { - name = "pin-project"; - version = "1.0.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"; }; - dependencies = { - pin_project_internal = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.12" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.12" = overridableMkRustCrate (profileName: rec { - name = "pin-project-internal"; - version = "1.0.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" = overridableMkRustCrate (profileName: rec { - name = "pin-project-lite"; - version = "0.2.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "pin-utils"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.26" = overridableMkRustCrate (profileName: rec { - name = "pkg-config"; - version = "0.3.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_base"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - no_std_net = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_datalink"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" { inherit profileName; }).out; - pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" = overridableMkRustCrate (profileName: rec { - name = "pnet_sys"; - version = "0.33.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ppv-lite86."0.2.17" = overridableMkRustCrate (profileName: rec { - name = "ppv-lite86"; - version = "0.2.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"; }; - features = builtins.concatLists [ - [ "simd" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "pretty_env_logger"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"; }; - dependencies = { - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".env_logger."0.7.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-error"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "syn" ] - [ "syn-error" ] - ]; - dependencies = { - proc_macro_error_attr = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error-attr."1.0.4" { profileName = "__noProfile"; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-error-attr."1.0.4" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-error-attr"; - version = "1.0.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - }; - buildDependencies = { - version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro-hack."0.5.20+deprecated" = overridableMkRustCrate (profileName: rec { - name = "proc-macro-hack"; - version = "0.5.20+deprecated"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" = overridableMkRustCrate (profileName: rec { - name = "proc-macro2"; - version = "1.0.50"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - [ "span-locations" ] - ]; - dependencies = { - unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.3" = overridableMkRustCrate (profileName: rec { - name = "prometheus"; - version = "0.13.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus") "default") - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus") "protobuf") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "fnv" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "memchr" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "protobuf" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" then "thiserror" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-derive") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-build"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "heck" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "itertools" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "multimap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "petgraph" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_types" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "regex" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tempfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tempfile."3.3.0" { inherit profileName; }).out; - }; - buildDependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "which" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".which."4.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-derive"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "anyhow" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.68" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "itertools" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" = overridableMkRustCrate (profileName: rec { - name = "prost-types"; - version = "0.9.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".protobuf."2.28.0" = overridableMkRustCrate (profileName: rec { - name = "protobuf"; - version = "2.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".quick-error."1.2.3" = overridableMkRustCrate (profileName: rec { - name = "quick-error"; - version = "1.2.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" = overridableMkRustCrate (profileName: rec { - name = "quick-xml"; - version = "0.21.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0452695941410a58c8ce4391707ba9bad26a247173bd9886a05a5e8a8babec75"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - [ "serialize" ] - ]; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" = overridableMkRustCrate (profileName: rec { - name = "quote"; - version = "1.0.23"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec { - name = "rand"; - version = "0.8.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "getrandom" ] - [ "libc" ] - [ "rand_chacha" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "small_rng") - [ "std" ] - [ "std_rng" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - rand_chacha = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" { inherit profileName; }).out; - rand_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" = overridableMkRustCrate (profileName: rec { - name = "rand_chacha"; - version = "0.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - ppv_lite86 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ppv-lite86."0.2.17" { inherit profileName; }).out; - rand_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" = overridableMkRustCrate (profileName: rec { - name = "rand_core"; - version = "0.6.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "getrandom" ] - [ "std" ] - ]; - dependencies = { - getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" = overridableMkRustCrate (profileName: rec { - name = "redox_syscall"; - version = "0.2.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"; }; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".redox_users."0.4.3" = overridableMkRustCrate (profileName: rec { - name = "redox_users"; - version = "0.4.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"; }; - dependencies = { - getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - syscall = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - thiserror = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" = overridableMkRustCrate (profileName: rec { - name = "regex"; - version = "1.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"; }; - features = builtins.concatLists [ - [ "aho-corasick" ] - [ "default" ] - [ "memchr" ] - [ "perf" ] - [ "perf-cache" ] - [ "perf-dfa" ] - [ "perf-inline" ] - [ "perf-literal" ] - [ "std" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - ]; - dependencies = { - aho_corasick = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.20" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.1.10" = overridableMkRustCrate (profileName: rec { - name = "regex-automata"; - version = "0.1.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"; }; - features = builtins.concatLists [ - [ "default" ] - [ "regex-syntax" ] - [ "std" ] - ]; - dependencies = { - regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.28" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.28" = overridableMkRustCrate (profileName: rec { - name = "regex-syntax"; - version = "0.6.28"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"; }; - features = builtins.concatLists [ - [ "default" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".remove_dir_all."0.5.3" = overridableMkRustCrate (profileName: rec { - name = "remove_dir_all"; - version = "0.5.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"; }; - dependencies = { - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.14" = overridableMkRustCrate (profileName: rec { - name = "reqwest"; - version = "0.11.14"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "__rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "__tls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "hyper-rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "json") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls-pemfile") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "rustls-tls-manual-roots") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "serde_json") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "tokio-rustls") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "encoding_rs" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.31" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "h2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.15" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "hyper_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "ipnet" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.7.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "mime" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mime."0.3.16" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "rustls_pemfile" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_urlencoded."0.7.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && !(hostPlatform.parsed.cpu.name == "wasm32") then "tokio_rustls" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.23.4" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "url" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.3.1" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "wasm_bindgen_futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-futures."0.4.33" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.parsed.cpu.name == "wasm32" then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.60" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && hostPlatform.isWindows then "winreg" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winreg."0.10.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" = overridableMkRustCrate (profileName: rec { - name = "ring"; - version = "0.16.20"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "dev_urandom_fallback" ] - [ "once_cell" ] - ]; - dependencies = { - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.60" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rmp."0.8.11" = overridableMkRustCrate (profileName: rec { - name = "rmp"; - version = "0.8.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.15" { inherit profileName; }).out; - paste = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".paste."1.0.11" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" = overridableMkRustCrate (profileName: rec { - name = "rmp-serde"; - version = "0.15.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d"; }; - dependencies = { - byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out; - rmp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp."0.8.11" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" = overridableMkRustCrate (profileName: rec { - name = "roxmltree"; - version = "0.14.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - xmlparser = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" = overridableMkRustCrate (profileName: rec { - name = "rusoto_core"; - version = "0.48.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"; }; - features = builtins.concatLists [ - [ "hyper-rustls" ] - [ "rustls" ] - ]; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - hyper_rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-rustls."0.23.2" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - rusoto_credential = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" { inherit profileName; }).out; - rusoto_signature = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_signature."0.48.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - xml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xml-rs."0.8.4" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" = overridableMkRustCrate (profileName: rec { - name = "rusoto_credential"; - version = "0.48.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"; }; - dependencies = { - async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - dirs_next = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - shlex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".shlex."1.1.0" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - zeroize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rusoto_signature."0.48.0" = overridableMkRustCrate (profileName: rec { - name = "rusoto_signature"; - version = "0.48.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"; }; - dependencies = { - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" { inherit profileName; }).out; - futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.25" { inherit profileName; }).out; - hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; - hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.11.0" { inherit profileName; }).out; - http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - rusoto_credential = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - }; - buildDependencies = { - rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.28.0" = overridableMkRustCrate (profileName: rec { - name = "rusqlite"; - version = "0.28.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled") - (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.25.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" = overridableMkRustCrate (profileName: rec { - name = "rustc-demangle"; - version = "0.1.21"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "rustc_version"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"; }; - dependencies = { - semver = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".semver."1.0.16" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustix."0.36.7" = overridableMkRustCrate (profileName: rec { - name = "rustix"; - version = "0.36.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "default") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "io-lifetimes") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "libc") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "std") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "termios") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "use-libc-auxv") - ]; - dependencies = { - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "powerpc64" || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "mips" || hostPlatform.parsed.cpu.name == "mips64"))) then "libc_errno" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".errno."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "io_lifetimes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".io-lifetimes."1.0.4" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && (hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "powerpc64" || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "mips" || hostPlatform.parsed.cpu.name == "mips64")) || !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "powerpc64" || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "mips" || hostPlatform.parsed.cpu.name == "mips64")))) then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && (hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "powerpc64" || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "mips" || hostPlatform.parsed.cpu.name == "mips64")) || (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux") && !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.significantByte == "littleEndian" && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "powerpc64" || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "mips" || hostPlatform.parsed.cpu.name == "mips64")))) then "linux_raw_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.1.4" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") && hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls."0.19.1" = overridableMkRustCrate (profileName: rec { - name = "rustls"; - version = "0.19.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"; }; - features = builtins.concatLists [ - [ "default" ] - [ "log" ] - [ "logging" ] - ]; - dependencies = { - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - sct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sct."0.6.1" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.21.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" = overridableMkRustCrate (profileName: rec { - name = "rustls"; - version = "0.20.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/reqwest") "dangerous_configuration") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - [ "log" ] - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - sct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sct."0.7.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "rustls-native-certs"; - version = "0.5.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"; }; - features = builtins.concatLists [ - [ "default" ] - [ "rustls" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix && !(hostPlatform.parsed.kernel.name == "darwin") then "openssl_probe" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-probe."0.1.5" { inherit profileName; }).out; - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.19.1" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "schannel" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schannel."0.1.21" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "darwin" then "security_framework" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".security-framework."2.8.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-native-certs."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "rustls-native-certs"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"; }; - dependencies = { - ${ if hostPlatform.isUnix && !(hostPlatform.parsed.kernel.name == "darwin") then "openssl_probe" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-probe."0.1.5" { inherit profileName; }).out; - rustls_pemfile = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.2" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "schannel" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schannel."0.1.21" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "darwin" then "security_framework" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".security-framework."2.8.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustls-pemfile."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "rustls-pemfile"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"; }; - dependencies = { - base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.11" = overridableMkRustCrate (profileName: rec { - name = "rustversion"; - version = "1.0.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.12" = overridableMkRustCrate (profileName: rec { - name = "ryu"; - version = "1.0.12"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" = overridableMkRustCrate (profileName: rec { - name = "same-file"; - version = "1.0.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"; }; - dependencies = { - ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schannel."0.1.21" = overridableMkRustCrate (profileName: rec { - name = "schannel"; - version = "0.1.21"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"; }; - dependencies = { - windows_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" = overridableMkRustCrate (profileName: rec { - name = "schemars"; - version = "0.8.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2a5fb6c61f29e723026dc8e923d94c694313212abbecbbe5f55a7748eec5b307"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "derive") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars") "schemars_derive") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "dyn_clone" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".dyn-clone."1.0.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars_derive."0.8.11" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".schemars_derive."0.8.11" = overridableMkRustCrate (profileName: rec { - name = "schemars_derive"; - version = "0.8.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f188d036977451159430f3b8dc82ec76364a42b7e289c2b18a9a18f4470058e9"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "serde_derive_internals" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive_internals."0.26.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "scopeguard"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".scratch."1.0.3" = overridableMkRustCrate (profileName: rec { - name = "scratch"; - version = "1.0.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sct."0.6.1" = overridableMkRustCrate (profileName: rec { - name = "sct"; - version = "0.6.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"; }; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sct."0.7.0" = overridableMkRustCrate (profileName: rec { - name = "sct"; - version = "0.7.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"; }; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".secrecy."0.8.0" = overridableMkRustCrate (profileName: rec { - name = "secrecy"; - version = "0.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zeroize" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.7" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".security-framework."2.8.2" = overridableMkRustCrate (profileName: rec { - name = "security-framework"; - version = "2.8.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"; }; - features = builtins.concatLists [ - [ "OSX_10_9" ] - [ "default" ] - ]; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - core_foundation = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation."0.9.3" { inherit profileName; }).out; - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.3" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - security_framework_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".security-framework-sys."2.8.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".security-framework-sys."2.8.0" = overridableMkRustCrate (profileName: rec { - name = "security-framework-sys"; - version = "2.8.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"; }; - features = builtins.concatLists [ - [ "OSX_10_9" ] - ]; - dependencies = { - core_foundation_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.3" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".semver."1.0.16" = overridableMkRustCrate (profileName: rec { - name = "semver"; - version = "1.0.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" = overridableMkRustCrate (profileName: rec { - name = "serde"; - version = "1.0.152"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "derive" ] - [ "rc" ] - [ "serde_derive" ] - [ "std" ] - ]; - dependencies = { - serde_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.152" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde-value."0.7.0" = overridableMkRustCrate (profileName: rec { - name = "serde-value"; - version = "0.7.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ordered_float" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ordered-float."2.10.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.8" = overridableMkRustCrate (profileName: rec { - name = "serde_bytes"; - version = "0.11.8"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.152" = overridableMkRustCrate (profileName: rec { - name = "serde_derive"; - version = "1.0.152"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_derive_internals."0.26.0" = overridableMkRustCrate (profileName: rec { - name = "serde_derive_internals"; - version = "0.26.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" = overridableMkRustCrate (profileName: rec { - name = "serde_json"; - version = "1.0.91"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc") - [ "default" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "indexmap") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "preserve_order") - [ "std" ] - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" { inherit profileName; }).out; - ryu = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.12" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.0" = overridableMkRustCrate (profileName: rec { - name = "serde_spanned"; - version = "0.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8"; }; - features = builtins.concatLists [ - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_urlencoded."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "serde_urlencoded"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "form_urlencoded" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "itoa" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "ryu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".serde_yaml."0.8.26" = overridableMkRustCrate (profileName: rec { - name = "serde_yaml"; - version = "0.8.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ryu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "yaml_rust" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".yaml-rust."0.4.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sha1."0.10.5" = overridableMkRustCrate (profileName: rec { - name = "sha1"; - version = "0.10.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.5" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" = overridableMkRustCrate (profileName: rec { - name = "sha2"; - version = "0.9.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - block_buffer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.9.0" { inherit profileName; }).out; - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "i686" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.5" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" { inherit profileName; }).out; - opaque_debug = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opaque-debug."0.3.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.6" = overridableMkRustCrate (profileName: rec { - name = "sha2"; - version = "0.10.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "i686" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.5" { inherit profileName; }).out; - digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sharded-slab."0.1.4" = overridableMkRustCrate (profileName: rec { - name = "sharded-slab"; - version = "0.1.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"; }; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".shlex."1.1.0" = overridableMkRustCrate (profileName: rec { - name = "shlex"; - version = "1.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" = overridableMkRustCrate (profileName: rec { - name = "signal-hook-registry"; - version = "1.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".siphasher."0.3.10" = overridableMkRustCrate (profileName: rec { - name = "siphasher"; - version = "0.3.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".slab."0.4.7" = overridableMkRustCrate (profileName: rec { - name = "slab"; - version = "0.4.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" = overridableMkRustCrate (profileName: rec { - name = "sled"; - version = "0.34.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"; }; - features = builtins.concatLists [ - [ "default" ] - [ "no_metrics" ] - ]; - dependencies = { - crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out; - crossbeam_epoch = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.13" { inherit profileName; }).out; - crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.14" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows" then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out; - fxhash = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - parking_lot = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" = overridableMkRustCrate (profileName: rec { - name = "smallvec"; - version = "1.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.7" = overridableMkRustCrate (profileName: rec { - name = "socket2"; - version = "0.4.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"; }; - features = builtins.concatLists [ - [ "all" ] - ]; - dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" = overridableMkRustCrate (profileName: rec { - name = "spin"; - version = "0.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".spin."0.9.4" = overridableMkRustCrate (profileName: rec { - name = "spin"; - version = "0.9.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"; }; - features = builtins.concatLists [ - [ "mutex" ] - [ "spin_mutex" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" = overridableMkRustCrate (profileName: rec { - name = "static_init"; - version = "1.0.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8a2a1c578e98c1c16fc3b8ec1328f7659a500737d7a0c6d625e73e830ff9c1f6"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out; - ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.6" { inherit profileName; }).out; - static_init_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - buildDependencies = { - cfg_aliases = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" = overridableMkRustCrate (profileName: rec { - name = "static_init_macro"; - version = "1.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "70a2595fc3aa78f2d0e45dd425b22282dd863273761cc77780914b2cf3003acf"; }; - dependencies = { - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - buildDependencies = { - cfg_aliases = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".strsim."0.10.0" = overridableMkRustCrate (profileName: rec { - name = "strsim"; - version = "0.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" = overridableMkRustCrate (profileName: rec { - name = "structopt"; - version = "0.3.26"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"; }; - dependencies = { - clap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."2.34.0" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - structopt_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt-derive."0.4.18" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".structopt-derive."0.4.18" = overridableMkRustCrate (profileName: rec { - name = "structopt-derive"; - version = "0.4.18"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"; }; - dependencies = { - heck = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }).out; - proc_macro_error = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro-error."1.0.4" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" = overridableMkRustCrate (profileName: rec { - name = "subtle"; - version = "2.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" = overridableMkRustCrate (profileName: rec { - name = "syn"; - version = "1.0.107"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"; }; - features = builtins.concatLists [ - [ "clone-impls" ] - [ "default" ] - [ "derive" ] - [ "extra-traits" ] - [ "full" ] - [ "parsing" ] - [ "printing" ] - [ "proc-macro" ] - [ "quote" ] - [ "visit" ] - [ "visit-mut" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.1" = overridableMkRustCrate (profileName: rec { - name = "synchronoise"; - version = "1.0.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"; }; - dependencies = { - ${ if rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb" then "crossbeam_queue" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.3.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".synstructure."0.12.6" = overridableMkRustCrate (profileName: rec { - name = "synstructure"; - version = "0.12.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "proc-macro" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - unicode_xid = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-xid."0.2.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" = overridableMkRustCrate (profileName: rec { - name = "systemstat"; - version = "0.2.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2"; }; - dependencies = { - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out; - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "nom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" { inherit profileName; }).out; - time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.17" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tempfile."3.3.0" = overridableMkRustCrate (profileName: rec { - name = "tempfile"; - version = "3.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "fastrand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.8.0" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && (hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi") then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.16" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "remove_dir_all" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".remove_dir_all."0.5.3" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".termcolor."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "termcolor"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"; }; - dependencies = { - ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".textwrap."0.11.0" = overridableMkRustCrate (profileName: rec { - name = "textwrap"; - version = "0.11.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"; }; - dependencies = { - unicode_width = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.38" = overridableMkRustCrate (profileName: rec { - name = "thiserror"; - version = "1.0.38"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"; }; - dependencies = { - thiserror_impl = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror-impl."1.0.38" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thiserror-impl."1.0.38" = overridableMkRustCrate (profileName: rec { - name = "thiserror-impl"; - version = "1.0.38"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".thread_local."1.1.4" = overridableMkRustCrate (profileName: rec { - name = "thread_local"; - version = "1.1.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"; }; - dependencies = { - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time."0.1.45" = overridableMkRustCrate (profileName: rec { - name = "time"; - version = "0.1.45"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"; }; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if hostPlatform.parsed.kernel.name == "wasi" then "wasi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasi."0.10.0+wasi-snapshot-preview1" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time."0.3.17" = overridableMkRustCrate (profileName: rec { - name = "time"; - version = "0.3.17"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "parsing" ] - [ "std" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - time_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.0" { inherit profileName; }).out; - time_macros = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".time-macros."0.2.6" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "time-core"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".time-macros."0.2.6" = overridableMkRustCrate (profileName: rec { - name = "time-macros"; - version = "0.2.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"; }; - features = builtins.concatLists [ - [ "parsing" ] - ]; - dependencies = { - time_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time-core."0.1.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.1" = overridableMkRustCrate (profileName: rec { - name = "timeago"; - version = "0.4.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5082dc942361cdfb74eab98bf995762d6015e5bb3a20bf7c5c71213778b4fcb4"; }; - features = builtins.concatLists [ - [ "chrono" ] - [ "default" ] - [ "isolang" ] - [ "translations" ] - ]; - dependencies = { - chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.23" { inherit profileName; }).out; - isolang = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".isolang."2.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tinyvec."1.6.0" = overridableMkRustCrate (profileName: rec { - name = "tinyvec"; - version = "1.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "tinyvec_macros" ] - ]; - dependencies = { - tinyvec_macros = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tinyvec_macros."0.1.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tinyvec_macros."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "tinyvec_macros"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" = overridableMkRustCrate (profileName: rec { - name = "tokio"; - version = "1.25.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"; }; - features = builtins.concatLists [ - [ "bytes" ] - [ "default" ] - [ "fs" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "full") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "k2v-client/cli") "io-std") - [ "io-util" ] - [ "libc" ] - [ "macros" ] - [ "memchr" ] - [ "mio" ] - [ "net" ] - [ "num_cpus" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "parking_lot") - [ "process" ] - [ "rt" ] - [ "rt-multi-thread" ] - [ "signal" ] - [ "signal-hook-registry" ] - [ "socket2" ] - [ "sync" ] - [ "time" ] - [ "tokio-macros" ] - [ "windows-sys" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.5.0" { inherit profileName; }).out; - mio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.5" { inherit profileName; }).out; - num_cpus = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.15.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }).out; - ${ if !(hostPlatform.parsed.cpu.name == "wasm32" || hostPlatform.parsed.cpu.name == "wasm64") then "socket2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.7" { inherit profileName; }).out; - tokio_macros = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-macros."1.8.2" { profileName = "__noProfile"; }).out; - ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" { inherit profileName; }).out; - }; - buildDependencies = { - autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-io-timeout."1.2.0" = overridableMkRustCrate (profileName: rec { - name = "tokio-io-timeout"; - version = "1.2.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-macros."1.8.2" = overridableMkRustCrate (profileName: rec { - name = "tokio-macros"; - version = "1.8.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.22.0" = overridableMkRustCrate (profileName: rec { - name = "tokio-rustls"; - version = "0.22.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"; }; - dependencies = { - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.19.1" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.21.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-rustls."0.23.4" = overridableMkRustCrate (profileName: rec { - name = "tokio-rustls"; - version = "0.23.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - [ "logging" ] - [ "tls12" ] - ]; - dependencies = { - rustls = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustls."0.20.8" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - webpki = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" = overridableMkRustCrate (profileName: rec { - name = "tokio-stream"; - version = "0.1.11"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"; }; - features = builtins.concatLists [ - [ "default" ] - [ "net" ] - [ "time" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.10" = overridableMkRustCrate (profileName: rec { - name = "tokio-util"; - version = "0.6.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "codec") - [ "default" ] - [ "io" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" = overridableMkRustCrate (profileName: rec { - name = "tokio-util"; - version = "0.7.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"; }; - features = builtins.concatLists [ - [ "codec" ] - [ "compat" ] - [ "default" ] - [ "futures-io" ] - [ "io" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "slab") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "time") - [ "tracing" ] - ]; - dependencies = { - bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_io = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.25" { inherit profileName; }).out; - futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.25" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "slab" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml."0.6.0" = overridableMkRustCrate (profileName: rec { - name = "toml"; - version = "0.6.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217"; }; - features = builtins.concatLists [ - [ "default" ] - [ "display" ] - [ "parse" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_spanned = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.0" { inherit profileName; }).out; - toml_datetime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" { inherit profileName; }).out; - toml_edit = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_edit."0.18.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" = overridableMkRustCrate (profileName: rec { - name = "toml_datetime"; - version = "0.5.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5"; }; - features = builtins.concatLists [ - [ "serde" ] - ]; - dependencies = { - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".toml_edit."0.18.1" = overridableMkRustCrate (profileName: rec { - name = "toml_edit"; - version = "0.18.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b"; }; - features = builtins.concatLists [ - [ "default" ] - [ "serde" ] - ]; - dependencies = { - indexmap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - nom8 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom8."0.2.0" { inherit profileName; }).out; - serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.152" { inherit profileName; }).out; - serde_spanned = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_spanned."0.6.0" { inherit profileName; }).out; - toml_datetime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml_datetime."0.5.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "tonic"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "async-trait") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "codegen") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "h2") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "hyper") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "hyper-timeout") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-derive") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost1") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tokio") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tower") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "tracing-futures") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "transport") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_stream" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "async_trait" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.63" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "h2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.15" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "hyper" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "hyper_timeout" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "percent_encoding" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost1" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio_stream" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.11" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.10" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower_layer" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing_futures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" = overridableMkRustCrate (profileName: rec { - name = "tonic-build"; - version = "0.6.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "prost-build") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "transport") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "prost_build" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower."0.4.13" = overridableMkRustCrate (profileName: rec { - name = "tower"; - version = "0.4.13"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"; }; - features = builtins.concatLists [ - [ "__common" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "balance") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "buffer") - [ "default" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "discover") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "filter") - [ "futures-core" ] - [ "futures-util" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "indexmap") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "limit") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "load") - [ "log" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "make") - [ "pin-project" ] - [ "pin-project-lite" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "rand") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "ready-cache") - [ "retry" ] - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "slab") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "timeout") - [ "tokio" ] - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tokio-util") - [ "tracing" ] - [ "util" ] - ]; - dependencies = { - futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "indexmap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.9.2" { inherit profileName; }).out; - pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "slab" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.7" { inherit profileName; }).out; - tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.25.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tokio_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.4" { inherit profileName; }).out; - tower_layer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - tower_service = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-http."0.3.5" = overridableMkRustCrate (profileName: rec { - name = "tower-http"; - version = "0.3.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "auth") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "base64") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "map-response-body") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "trace") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "tracing") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "base64" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bytes" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "futures_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.25" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_body" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.5" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "http_range_header" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-range-header."0.3.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "pin_project_lite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_layer" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tower_service" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "tower-layer"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.2" = overridableMkRustCrate (profileName: rec { - name = "tower-service"; - version = "0.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" = overridableMkRustCrate (profileName: rec { - name = "tracing"; - version = "0.1.37"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"; }; - features = builtins.concatLists [ - [ "attributes" ] - [ "default" ] - [ "log" ] - [ "std" ] - [ "tracing-attributes" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - pin_project_lite = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.9" { inherit profileName; }).out; - tracing_attributes = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.23" { profileName = "__noProfile"; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.30" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.23" = overridableMkRustCrate (profileName: rec { - name = "tracing-attributes"; - version = "0.1.23"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"; }; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.30" = overridableMkRustCrate (profileName: rec { - name = "tracing-core"; - version = "0.1.30"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"; }; - features = builtins.concatLists [ - [ "default" ] - [ "once_cell" ] - [ "std" ] - [ "valuable" ] - ]; - dependencies = { - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - ${ if false then "valuable" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".valuable."0.1.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" = overridableMkRustCrate (profileName: rec { - name = "tracing-futures"; - version = "0.2.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "default") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "pin-project") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std") - (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") "std-future") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "pin_project" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.12" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-log."0.1.3" = overridableMkRustCrate (profileName: rec { - name = "tracing-log"; - version = "0.1.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"; }; - features = builtins.concatLists [ - [ "log-tracer" ] - [ "std" ] - ]; - dependencies = { - lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.30" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.16" = overridableMkRustCrate (profileName: rec { - name = "tracing-subscriber"; - version = "0.3.16"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "ansi" ] - [ "default" ] - [ "env-filter" ] - [ "fmt" ] - [ "matchers" ] - [ "nu-ansi-term" ] - [ "once_cell" ] - [ "regex" ] - [ "registry" ] - [ "sharded-slab" ] - [ "smallvec" ] - [ "std" ] - [ "thread_local" ] - [ "tracing" ] - [ "tracing-log" ] - ]; - dependencies = { - matchers = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".matchers."0.1.0" { inherit profileName; }).out; - nu_ansi_term = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nu-ansi-term."0.46.0" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.7.1" { inherit profileName; }).out; - sharded_slab = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sharded-slab."0.1.4" { inherit profileName; }).out; - smallvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.10.0" { inherit profileName; }).out; - thread_local = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thread_local."1.1.4" { inherit profileName; }).out; - tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out; - tracing_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.30" { inherit profileName; }).out; - tracing_log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-log."0.1.3" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" = overridableMkRustCrate (profileName: rec { - name = "treediff"; - version = "3.0.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "serde_json") - (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "with-serde-json") - ]; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "serde_json" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.91" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".try-lock."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "try-lock"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".typenum."1.16.0" = overridableMkRustCrate (profileName: rec { - name = "typenum"; - version = "1.16.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-bidi."0.3.10" = overridableMkRustCrate (profileName: rec { - name = "unicode-bidi"; - version = "0.3.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"; }; - features = builtins.concatLists [ - [ "default" ] - [ "hardcoded-data" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.6" = overridableMkRustCrate (profileName: rec { - name = "unicode-ident"; - version = "1.0.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-normalization."0.1.22" = overridableMkRustCrate (profileName: rec { - name = "unicode-normalization"; - version = "0.1.22"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - dependencies = { - tinyvec = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tinyvec."1.6.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-segmentation."1.10.0" = overridableMkRustCrate (profileName: rec { - name = "unicode-segmentation"; - version = "1.10.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-width."0.1.10" = overridableMkRustCrate (profileName: rec { - name = "unicode-width"; - version = "0.1.10"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".unicode-xid."0.2.4" = overridableMkRustCrate (profileName: rec { - name = "unicode-xid"; - version = "0.2.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" = overridableMkRustCrate (profileName: rec { - name = "untrusted"; - version = "0.7.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".url."2.3.1" = overridableMkRustCrate (profileName: rec { - name = "url"; - version = "2.3.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"; }; - features = builtins.concatLists [ - [ "default" ] - ]; - dependencies = { - form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.1.0" { inherit profileName; }).out; - idna = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.3.0" { inherit profileName; }).out; - percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.2.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".uuid."1.2.2" = overridableMkRustCrate (profileName: rec { - name = "uuid"; - version = "1.2.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"; }; - features = builtins.concatLists [ - [ "default" ] - [ "getrandom" ] - [ "rng" ] - [ "std" ] - [ "v4" ] - ]; - dependencies = { - getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.8" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".valuable."0.1.0" = overridableMkRustCrate (profileName: rec { - name = "valuable"; - version = "0.1.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" = overridableMkRustCrate (profileName: rec { - name = "vcpkg"; - version = "0.2.15"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" = overridableMkRustCrate (profileName: rec { - name = "version_check"; - version = "0.9.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".walkdir."2.3.2" = overridableMkRustCrate (profileName: rec { - name = "walkdir"; - version = "2.3.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"; }; - dependencies = { - same_file = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".want."0.3.0" = overridableMkRustCrate (profileName: rec { - name = "want"; - version = "0.3.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"; }; - dependencies = { - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - try_lock = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".try-lock."0.2.4" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasi."0.10.0+wasi-snapshot-preview1" = overridableMkRustCrate (profileName: rec { - name = "wasi"; - version = "0.10.0+wasi-snapshot-preview1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasi."0.11.0+wasi-snapshot-preview1" = overridableMkRustCrate (profileName: rec { - name = "wasi"; - version = "0.11.0+wasi-snapshot-preview1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen"; - version = "0.2.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"; }; - features = builtins.concatLists [ - [ "default" ] - [ "spans" ] - [ "std" ] - ]; - dependencies = { - cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - wasm_bindgen_macro = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro."0.2.83" { profileName = "__noProfile"; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-backend."0.2.83" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-backend"; - version = "0.2.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - bumpalo = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bumpalo."3.12.0" { inherit profileName; }).out; - log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.17" { inherit profileName; }).out; - once_cell = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - wasm_bindgen_shared = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.83" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-futures."0.4.33" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-futures"; - version = "0.4.33"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "js_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "wasm_bindgen" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") && builtins.elem "atomics" hostPlatformFeatures then "web_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.60" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro."0.2.83" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-macro"; - version = "0.2.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - wasm_bindgen_macro_support = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro-support."0.2.83" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-macro-support."0.2.83" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-macro-support"; - version = "0.2.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"; }; - features = builtins.concatLists [ - [ "spans" ] - ]; - dependencies = { - proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.50" { inherit profileName; }).out; - quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.23" { inherit profileName; }).out; - syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.107" { inherit profileName; }).out; - wasm_bindgen_backend = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-backend."0.2.83" { inherit profileName; }).out; - wasm_bindgen_shared = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.83" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen-shared."0.2.83" = overridableMkRustCrate (profileName: rec { - name = "wasm-bindgen-shared"; - version = "0.2.83"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.60" = overridableMkRustCrate (profileName: rec { - name = "web-sys"; - version = "0.3.60"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Blob") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "BlobPropertyBag") - [ "Crypto" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Event") - [ "EventTarget" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "File") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "FormData") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Headers") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "MessageEvent") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "ReadableStream") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Request") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestCredentials") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestInit") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "RequestMode") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Response") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "ServiceWorkerGlobalScope") - [ "Window" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "Worker") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "WorkerGlobalScope") - ]; - dependencies = { - js_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.60" { inherit profileName; }).out; - wasm_bindgen = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wasm-bindgen."0.2.83" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".webpki."0.21.4" = overridableMkRustCrate (profileName: rec { - name = "webpki"; - version = "0.21.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - [ "trust_anchor_util" ] - ]; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".webpki."0.22.0" = overridableMkRustCrate (profileName: rec { - name = "webpki"; - version = "0.22.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "std" ] - ]; - dependencies = { - ring = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ring."0.16.20" { inherit profileName; }).out; - untrusted = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".which."4.4.0" = overridableMkRustCrate (profileName: rec { - name = "which"; - version = "4.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"; }; - dependencies = { - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "either" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.8.1" { inherit profileName; }).out; - ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - ${ if (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp") && hostPlatform.isWindows then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.17.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" = overridableMkRustCrate (profileName: rec { - name = "winapi"; - version = "0.3.9"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"; }; - features = builtins.concatLists [ - [ "activation" ] - [ "combaseapi" ] - [ "consoleapi" ] - [ "errhandlingapi" ] - [ "fileapi" ] - [ "handleapi" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "impl-debug") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "impl-default") - [ "knownfolders" ] - [ "minwinbase" ] - [ "minwindef" ] - [ "ntdef" ] - [ "ntsecapi" ] - [ "ntstatus" ] - [ "objbase" ] - [ "pdh" ] - [ "processenv" ] - [ "processthreadsapi" ] - [ "profileapi" ] - [ "roapi" ] - [ "shlobj" ] - [ "std" ] - [ "synchapi" ] - [ "sysinfoapi" ] - [ "timezoneapi" ] - [ "winbase" ] - [ "wincon" ] - [ "winerror" ] - [ "winnt" ] - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "winreg") - [ "winsock2" ] - [ "winstring" ] - [ "ws2def" ] - [ "ws2ipdef" ] - [ "ws2tcpip" ] - [ "wtypesbase" ] - ]; - dependencies = { - ${ if hostPlatform.config == "i686-pc-windows-gnu" then "winapi_i686_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" then "winapi_x86_64_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "winapi-i686-pc-windows-gnu"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.5" = overridableMkRustCrate (profileName: rec { - name = "winapi-util"; - version = "0.1.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"; }; - dependencies = { - ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { - name = "winapi-x86_64-pc-windows-gnu"; - version = "0.4.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.42.0" = overridableMkRustCrate (profileName: rec { - name = "windows-sys"; - version = "0.42.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"; }; - features = builtins.concatLists [ - [ "Win32" ] - [ "Win32_Foundation" ] - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "Win32_NetworkManagement") - (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "Win32_NetworkManagement_IpHelper") - [ "Win32_Networking" ] - [ "Win32_Networking_WinSock" ] - [ "Win32_Security" ] - [ "Win32_Security_Authentication" ] - [ "Win32_Security_Authentication_Identity" ] - [ "Win32_Security_Authorization" ] - [ "Win32_Security_Credentials" ] - [ "Win32_Security_Cryptography" ] - [ "Win32_Storage" ] - [ "Win32_Storage_FileSystem" ] - [ "Win32_System" ] - [ "Win32_System_Console" ] - [ "Win32_System_IO" ] - (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "Win32_System_LibraryLoader") - [ "Win32_System_Memory" ] - [ "Win32_System_Pipes" ] - [ "Win32_System_SystemServices" ] - [ "Win32_System_Threading" ] - [ "Win32_System_WindowsProgramming" ] - [ "default" ] - ]; - dependencies = { - ${ if hostPlatform.config == "aarch64-pc-windows-gnullvm" then "windows_aarch64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-gnullvm" then "windows_x86_64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.42.1" { inherit profileName; }).out; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.42.1" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_aarch64_gnullvm"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_aarch64_msvc"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_i686_gnu"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_i686_msvc"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_gnu"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_gnullvm"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.42.1" = overridableMkRustCrate (profileName: rec { - name = "windows_x86_64_msvc"; - version = "0.42.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".winreg."0.10.1" = overridableMkRustCrate (profileName: rec { - name = "winreg"; - version = "0.10.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"; }; - dependencies = { - ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".xml-rs."0.8.4" = overridableMkRustCrate (profileName: rec { - name = "xml-rs"; - version = "0.8.4"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"; }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" = overridableMkRustCrate (profileName: rec { - name = "xmlparser"; - version = "0.13.3"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.6" = overridableMkRustCrate (profileName: rec { - name = "xxhash-rust"; - version = "0.8.6"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"; }; - features = builtins.concatLists [ - [ "xxh3" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".yaml-rust."0.4.5" = overridableMkRustCrate (profileName: rec { - name = "yaml-rust"; - version = "0.4.5"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"; }; - dependencies = { - ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "linked_hash_map" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".linked-hash-map."0.5.6" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.7" = overridableMkRustCrate (profileName: rec { - name = "zeroize"; - version = "1.5.7"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"; }; - features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd."0.11.2+zstd.1.5.2" = overridableMkRustCrate (profileName: rec { - name = "zstd"; - version = "0.11.2+zstd.1.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"; }; - dependencies = { - zstd_safe = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."5.0.2+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd."0.12.2+zstd.1.5.2" = overridableMkRustCrate (profileName: rec { - name = "zstd"; - version = "0.12.2+zstd.1.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e9262a83dc741c0b0ffec209881b45dbc232c21b02a2b9cb1adb93266e41303d"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - ]; - dependencies = { - zstd_safe = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."6.0.2+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd-safe."5.0.2+zstd.1.5.2" = overridableMkRustCrate (profileName: rec { - name = "zstd-safe"; - version = "5.0.2+zstd.1.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"; }; - features = builtins.concatLists [ - [ "std" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - zstd_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-sys."2.0.5+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd-safe."6.0.2+zstd.1.5.2" = overridableMkRustCrate (profileName: rec { - name = "zstd-safe"; - version = "6.0.2+zstd.1.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "a6cf39f730b440bab43da8fb5faf5f254574462f73f260f85f7987f32154ff17"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - [ "std" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - zstd_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-sys."2.0.5+zstd.1.5.2" { inherit profileName; }).out; - }; - }); - - "registry+https://github.com/rust-lang/crates.io-index".zstd-sys."2.0.5+zstd.1.5.2" = overridableMkRustCrate (profileName: rec { - name = "zstd-sys"; - version = "2.0.5+zstd.1.5.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"; }; - features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "pkg-config") - [ "std" ] - ]; - dependencies = { - libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - }; - buildDependencies = { - cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.79" { profileName = "__noProfile"; }).out; - pkg_config = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.26" { profileName = "__noProfile"; }).out; - }; - }); - -} diff --git a/Cargo.toml b/Cargo.toml index a9fd4423..df4005a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,20 +3,151 @@ resolver = "2" members = [ "src/db", "src/util", + "src/net", "src/rpc", "src/table", "src/block", "src/model", - "src/api", + "src/api/common", + "src/api/s3", + "src/api/k2v", + "src/api/admin", "src/web", "src/garage", "src/k2v-client", + "src/format-table", ] default-members = ["src/garage"] -[profile.dev] -lto = "off" +[workspace.dependencies] + +# Internal Garage crates +format_table = { version = "0.1.1", path = "src/format-table" } +garage_api_common = { version = "1.3.1", path = "src/api/common" } +garage_api_admin = { version = "1.3.1", path = "src/api/admin" } +garage_api_s3 = { version = "1.3.1", path = "src/api/s3" } +garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" } +garage_block = { version = "1.3.1", path = "src/block" } +garage_db = { version = "1.3.1", path = "src/db", default-features = false } +garage_model = { version = "1.3.1", path = "src/model", default-features = false } +garage_net = { version = "1.3.1", path = "src/net" } +garage_rpc = { version = "1.3.1", path = "src/rpc" } +garage_table = { version = "1.3.1", path = "src/table" } +garage_util = { version = "1.3.1", path = "src/util" } +garage_web = { version = "1.3.1", path = "src/web" } +k2v-client = { version = "0.0.4", path = "src/k2v-client" } + +# External crates from crates.io +arc-swap = "1.0" +argon2 = "0.5" +async-trait = "0.1.7" +backtrace = "0.3" +base64 = "0.21" +blake2 = "0.10" +bytes = "1.0" +bytesize = "1.1" +cfg-if = "1.0" +chrono = "0.4" +crc32fast = "1.4" +crc32c = "0.6" +crypto-common = "0.1" +gethostname = "0.4" +git-version = "0.3.4" +hex = "0.4" +hexdump = "0.1" +hmac = "0.12" +itertools = "0.12" +ipnet = "2.9.0" +lazy_static = "1.4" +md-5 = "0.10" +mktemp = "0.5" +nix = { version = "0.29", default-features = false, features = ["fs"] } +nom = "7.1" +parking_lot = "0.12" +parse_duration = "2.1" +pin-project = "1.0.12" +pnet_datalink = "0.34" +rand = "0.8" +sha1 = "0.10" +sha2 = "0.10" +timeago = { version = "0.4", default-features = false } +xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } + +aes-gcm = { version = "0.10", features = ["aes", "stream"] } +sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } +kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] } + +clap = { version = "4.1", features = ["derive", "env"] } +pretty_env_logger = "0.5" +structopt = { version = "0.3", default-features = false } +syslog-tracing = "0.3" +tracing = "0.1" +tracing-journald = "0.3.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +heed = { version = "0.11", default-features = false, features = ["lmdb"] } +rusqlite = "0.37" +r2d2 = "0.8" +r2d2_sqlite = "0.31" +fjall = "2.4" + +async-compression = { version = "0.4", features = ["tokio", "zstd"] } +zstd = { version = "0.13", default-features = false } + +quick-xml = { version = "0.26", features = [ "serialize" ] } +rmp-serde = "1.1.2" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_bytes = "0.11" +serde_json = "1.0" +toml = { version = "0.8", default-features = false, features = ["parse"] } + +# newer version requires rust edition 2021 +k8s-openapi = { version = "0.21", features = ["v1_24"] } +kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] } +schemars = "0.8" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] } + +form_urlencoded = "1.0.0" +http = "1.0" +httpdate = "1.0" +http-range = "0.1" +http-body-util = "0.1" +hyper = { version = "1.0", default-features = false } +hyper-util = { version = "0.1", features = [ "full" ] } +multer = "3.0" +percent-encoding = "2.2" +roxmltree = "0.19" +url = "2.3" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +tokio-util = { version = "0.7", features = ["compat", "io"] } +tokio-stream = { version = "0.1", features = ["net"] } + +opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" +opentelemetry-contrib = "0.9" +prometheus = "0.13" + +# used by the k2v-client crate only +aws-sigv4 = { version = "1.1", default-features = false } +hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] } +log = "0.4" +thiserror = "2.0" + +# ---- used only as build / dev dependencies ---- +assert-json-diff = "2.0" +rustc_version = "0.4.0" +static_init = "1.0" +aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] } +aws-sdk-config = { version = "1.62", default-features = false } +aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] } [profile.release] -debug = true +lto = "thin" +codegen-units = 16 +opt-level = 3 +strip = "debuginfo" diff --git a/Dockerfile b/Dockerfile index 62e51b32..2e301ee9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,5 @@ FROM scratch ENV RUST_BACKTRACE=1 ENV RUST_LOG=garage=info -COPY result-bin/bin/garage / +COPY result/bin/garage / CMD [ "/garage", "server"] diff --git a/Makefile b/Makefile index 55fa16dd..35c3f22c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,8 @@ -.PHONY: doc all release shell run1 run2 run3 +.PHONY: doc all run1 run2 run3 all: - clear; cargo build - -release: - nix-build --attr pkgs.amd64.release --no-build-output - -shell: - nix-shell + clear + cargo build # ---- diff --git a/README.md b/README.md index 9992fff2..a0f082a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage) +Garage [![status-badge](https://woodpecker.deuxfleurs.fr/api/badges/1/status.svg)](https://woodpecker.deuxfleurs.fr/repos/1) ===

diff --git a/default.nix b/default.nix index 5090db6a..6974567f 100644 --- a/default.nix +++ b/default.nix @@ -3,61 +3,22 @@ with import ./nix/common.nix; let - pkgs = import pkgsSrc { }; + pkgs = import nixpkgs { }; compile = import ./nix/compile.nix; - build_debug_and_release = (target: { - debug = (compile { - inherit system target git_version pkgsSrc cargo2nixOverlay; - release = false; - }).workspace.garage { compileMode = "build"; }; - - release = (compile { - inherit system target git_version pkgsSrc cargo2nixOverlay; - release = true; - }).workspace.garage { compileMode = "build"; }; - }); - - test = (rustPkgs: - pkgs.symlinkJoin { - name = "garage-tests"; - paths = - builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) - (builtins.attrNames rustPkgs.workspace); - }); + build_release = target: (compile { + inherit target system git_version nixpkgs; + crane = flake.inputs.crane; + rust-overlay = flake.inputs.rust-overlay; + release = true; + }).garage; in { - pkgs = { - amd64 = build_debug_and_release "x86_64-unknown-linux-musl"; - i386 = build_debug_and_release "i686-unknown-linux-musl"; - arm64 = build_debug_and_release "aarch64-unknown-linux-musl"; - arm = build_debug_and_release "armv6l-unknown-linux-musleabihf"; - win = { - amd64 = build_debug_and_release "x86_64-w64-mingw32"; - }; - apple = { - arm64 = build_debug_and_release "aarch64-apple-darwin"; - amd64 = build_debug_and_release "x86_64-apple-darwin"; - }; - }; - test = { - amd64 = test (compile { - inherit system git_version pkgsSrc cargo2nixOverlay; - target = "x86_64-unknown-linux-musl"; - features = [ - "garage/bundled-libs" - "garage/k2v" - "garage/sled" - "garage/lmdb" - "garage/sqlite" - ]; - }); - }; - clippy = { - amd64 = (compile { - inherit system git_version pkgsSrc cargo2nixOverlay; - target = "x86_64-unknown-linux-musl"; - compiler = "clippy"; - }).workspace.garage { compileMode = "build"; }; + releasePackages = { + amd64 = build_release "x86_64-unknown-linux-musl"; + i386 = build_release "i686-unknown-linux-musl"; + arm64 = build_release "aarch64-unknown-linux-musl"; + arm = build_release "armv6l-unknown-linux-musleabihf"; }; + flakePackages = flake.packages.${system}; } diff --git a/doc/api/garage-admin-v0.yml b/doc/api/garage-admin-v0.yml index 51968894..83316d93 100644 --- a/doc/api/garage-admin-v0.yml +++ b/doc/api/garage-admin-v0.yml @@ -632,7 +632,7 @@ paths: operationId: "UpdateBucket" summary: "Update a bucket" description: | - All fields (`websiteAccess` and `quotas`) are optionnal. + All fields (`websiteAccess` and `quotas`) are optional. If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. diff --git a/doc/api/garage-admin-v1.html b/doc/api/garage-admin-v1.html new file mode 100644 index 00000000..783d459e --- /dev/null +++ b/doc/api/garage-admin-v1.html @@ -0,0 +1,24 @@ + + + + Garage Adminstration API v0 + + + + + + + + + + + + + diff --git a/doc/api/garage-admin-v1.yml b/doc/api/garage-admin-v1.yml new file mode 100644 index 00000000..a70dc97b --- /dev/null +++ b/doc/api/garage-admin-v1.yml @@ -0,0 +1,1362 @@ +openapi: 3.0.0 +info: + version: v0.9.0 + title: Garage Administration API v0+garage-v0.9.0 + description: | + Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks. + + *Disclaimer: The API is not stable yet, hence its v0 tag. The API can change at any time, and changes can include breaking backward compatibility. Read the changelog and upgrade your scripts before upgrading. Additionnaly, this specification is very early stage and can contain bugs, especially on error return codes/types that are not tested yet. Do not expect a well finished and polished product!* +paths: + /health: + get: + tags: + - Nodes + operationId: "GetHealth" + summary: "Cluster health report" + description: | + Returns the global status of the cluster, the number of connected nodes (over the number of known ones), the number of healthy storage nodes (over the declared ones), and the number of healthy partitions (over the total). + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Information about the queried node, its environment and the current layout + content: + application/json: + schema: + type: object + required: [ status, knownNodes, connectedNodes, storageNodes, storageNodesOk, partitions, partitionsQuorum, partitionsAllOk ] + properties: + status: + type: string + example: "healthy" + knownNodes: + type: integer + format: int64 + example: 4 + connectedNodes: + type: integer + format: int64 + example: 4 + storageNodes: + type: integer + format: int64 + example: 3 + storageNodesOk: + type: integer + format: int64 + example: 3 + partitions: + type: integer + format: int64 + example: 256 + partitionsQuorum: + type: integer + format: int64 + example: 256 + partitionsAllOk: + type: integer + format: int64 + example: 256 + /status: + get: + tags: + - Nodes + operationId: "GetNodes" + summary: "Describe cluster" + description: | + Returns the cluster's current status, including: + - ID of the node being queried and its version of the Garage daemon + - Live nodes + - Currently configured cluster layout + - Staged changes to the cluster layout + + *Capacity is given in bytes* + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Information about the queried node, its environment and the current layout + content: + application/json: + schema: + type: object + required: [ node, garageVersion, garageFeatures, rustVersion, dbEngine, knownNodes, layout ] + properties: + node: + type: string + example: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + garageVersion: + type: string + example: "v0.9.0" + garageFeatures: + type: array + items: + type: string + example: + - "k2v" + - "lmdb" + - "sqlite" + - "consul-discovery" + - "kubernetes-discovery" + - "metrics" + - "telemetry-otlp" + - "bundled-libs" + rustVersion: + type: string + example: "1.68.0" + dbEngine: + type: string + example: "LMDB (using Heed crate)" + knownNodes: + type: array + example: + - id: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + addr: "10.0.0.11:3901" + isUp: true + lastSeenSecsAgo: 9 + hostname: orion + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + addr: "10.0.0.12:3901" + isUp: true + lastSeenSecsAgo: 13 + hostname: pegasus + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + addr: "10.0.0.13:3901" + isUp: true + lastSeenSecsAgo: 2 + hostname: neptune + items: + $ref: '#/components/schemas/NodeNetworkInfo' + layout: + $ref: '#/components/schemas/ClusterLayout' + + /connect: + post: + tags: + - Nodes + operationId: "AddNode" + summary: "Connect a new node" + description: | + Instructs this Garage node to connect to other Garage nodes at specified `@`. `node_id` is generated automatically on node start. + requestBody: + required: true + content: + application/json: + schema: + type: array + example: + - "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901" + - "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901" + items: + type: string + + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '400': + description: | + Your request is malformed, check your JSON + '200': + description: | + The request has been handled correctly but it does not mean that all connection requests succeeded; some might have fail, you need to check the body! + content: + application/json: + schema: + type: array + example: + - success: true + error: + - success: false + error: "Handshake error" + items: + type: object + properties: + success: + type: boolean + example: true + error: + type: string + nullable: true + example: null + + /layout: + get: + tags: + - Layout + operationId: "GetLayout" + summary: "Details on the current and staged layout" + description: | + Returns the cluster's current layout, including: + - Currently configured cluster layout + - Staged changes to the cluster layout + + *Capacity is given in bytes* + *The info returned by this endpoint is a subset of the info returned by `GET /status`.* + responses: + '500': + description: | + The server can not answer your request because it is in a bad state + '200': + description: | + Returns the cluster's current cluster layout: + - Currently configured cluster layout + - Staged changes to the cluster layout + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterLayout' + + post: + tags: + - Layout + operationId: "AddLayout" + summary: "Send modifications to the cluster layout" + description: | + Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls of `GET /layout`. Once the set of staged changes is satisfactory, the user may call `POST /layout/apply` to apply the changed changes, or `POST /layout/revert` to clear all of the staged changes in the layout. + + Setting the capacity to `null` will configure the node as a gateway. + Otherwise, capacity must be now set in bytes (before Garage 0.9 it was arbitrary weights). + For example to declare 100GB, you must set `capacity: 100000000000`. + + Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes). + requestBody: + description: | + To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`). + To remove a node, simply pass the `remove: true` field. + This logic is represented in OpenAPI with a "One Of" object. + + Contrary to the CLI that may update only a subset of the fields capacity, zone and tags, when calling this API all of these values must be specified. + required: true + content: + application/json: + schema: + type: array + example: + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + zone: "geneva" + capacity: 100000000000 + tags: + - gateway + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + remove: true + items: + $ref: '#/components/schemas/NodeRoleChange' + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The layout modification has been correctly staged" + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterLayout' + + /layout/apply: + post: + tags: + - Layout + operationId: "ApplyLayout" + summary: "Apply staged layout" + description: | + Applies to the cluster the layout changes currently registered as staged layout changes. + + *Note: do not try to parse the `message` field of the response, it is given as an array of string specifically because its format is not stable.* + requestBody: + description: | + Similarly to the CLI, the body must include the version of the new layout that will be created, which MUST be 1 + the value of the currently existing layout in the cluster. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LayoutVersion' + + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The staged layout has been applied as the new layout of the cluster, a rebalance has been triggered." + content: + application/json: + schema: + type: object + required: [ message, layout ] + properties: + message: + type: array + items: + type: string + example: + - "==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====" + - "" + - "Partitions are replicated 1 times on at least 1 distinct zones." + - "" + - "Optimal partition size: 419.4 MB (3 B in previous layout)" + - "Usable capacity / total cluster capacity: 107.4 GB / 107.4 GB (100.0 %)" + - "Effective capacity (replication factor 1): 107.4 GB" + - "" + - "A total of 0 new copies of partitions need to be transferred." + - "" + - "dc1 Tags Partitions Capacity Usable capacity\n 6a8e08af2aab1083 a,v 256 (0 new) 107.4 GB 107.4 GB (100.0%)\n TOTAL 256 (256 unique) 107.4 GB 107.4 GB (100.0%)\n\n" + layout: + $ref: '#/components/schemas/ClusterLayout' + + + /layout/revert: + post: + tags: + - Layout + operationId: "RevertLayout" + summary: "Clear staged layout" + description: | + Clears all of the staged layout changes. + requestBody: + description: | + Reverting the staged changes is done by incrementing the version number and clearing the contents of the staged change list. Similarly to the CLI, the body must include the incremented version number, which MUST be 1 + the value of the currently existing layout in the cluster. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LayoutVersion' + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The staged layout has been cleared, you can start again sending modification from a fresh copy with `POST /layout`." + + "/key?list": + get: + tags: + - Key + operationId: "ListKeys" + summary: "List all keys" + description: | + Returns all API access keys in the cluster. + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string) + content: + application/json: + schema: + type: array + example: + - id: "GK31c2f218a2e44f485b94239e" + name: "test-key" + - id: "GKe10061ac9c2921f09e4c5540" + name: "" + items: + type: object + required: [ id ] + properties: + id: + type: string + name: + type: string + post: + tags: + - Key + operationId: "AddKey" + summary: "Create a new API key" + description: | + Creates a new API access key. + requestBody: + description: | + You can set a friendly name for this key. + If you don't want to, you can set the name to `null`. + + *Note: the secret key is returned in the response.* + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + nullable: true + example: "test-key" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The key has been added" + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + "/key": + get: + tags: + - Key + operationId: "GetKey" + summary: "Get key information" + description: | + Return information about a specific key like its identifiers, its permissions and buckets on which it has permissions. + You can search by specifying the exact key identifier (`id`) or by specifying a pattern (`search`). + + For confidentiality reasons, the secret key is not returned by default: you must pass the `showSecretKey` query parameter to get it. + parameters: + - name: id + in: query + description: | + The exact API access key generated by Garage. + + Incompatible with `search`. + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + - name: search + in: query + description: | + A pattern (beginning or full string) corresponding to a key identifier or friendly name. + + Incompatible with `id`. + example: "test-k" + schema: + type: string + - name: showSecretKey + in: query + schema: + type: string + default: "false" + enum: + - "true" + - "false" + example: "true" + required: false + description: "Wether or not the secret key should be returned in the response" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns information about the key + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + delete: + tags: + - Key + operationId: "DeleteKey" + summary: "Delete a key" + description: | + Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. + parameters: + - name: id + in: query + required: true + description: "The exact API access key generated by Garage" + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: "The key has been deleted" + + + post: + tags: + - Key + operationId: "UpdateKey" + summary: "Update a key" + description: | + Updates information about the specified API access key. + + *Note: the secret key is not returned in the response, `null` is sent instead.* + parameters: + - name: id + in: query + required: true + description: "The exact API access key generated by Garage" + example: "GK31c2f218a2e44f485b94239e" + schema: + type: string + requestBody: + description: | + For a given key, provide a first set with the permissions to grant, and a second set with the permissions to remove + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + example: "test-key" + allow: + type: object + example: + properties: + createBucket: + type: boolean + example: true + deny: + type: object + properties: + createBucket: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: | + Returns information about the key + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + + /key/import: + post: + tags: + - Key + operationId: "ImportKey" + summary: "Import an existing key" + description: | + Imports an existing API key. This feature must only be used for migrations and backup restore. + + **Do not use it to generate custom key identifiers or you will break your Garage cluster.** + requestBody: + description: | + Information on the key to import + required: true + content: + application/json: + schema: + type: object + required: [ name, accessKeyId, secretAccessKey ] + properties: + name: + type: string + example: "test-key" + nullable: true + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + secretAccessKey: + type: string + example: "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835" + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Invalid syntax or requested change" + '200': + description: "The key has been imported into the system" + content: + application/json: + schema: + $ref: '#/components/schemas/KeyInfo' + + "/bucket?list": + get: + tags: + - Bucket + operationId: "ListBuckets" + summary: "List all buckets" + description: | + List all the buckets on the cluster with their UUID and their global and local aliases. + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '200': + description: | + Returns the UUID of the bucket and all its aliases + content: + application/json: + schema: + type: array + example: + - id: "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033" + globalAliases: + - "container_registry" + - id: "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95" + localAliases: + - alias: "my_documents" + accessKeyid: "GK31c2f218a2e44f485b94239e" + - id: "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995" + globalAliases: + - "example.com" + - "www.example.com" + localAliases: + - alias: "corp_website" + accessKeyId: "GKe10061ac9c2921f09e4c5540" + - alias: "web" + accessKeyid: "GK31c2f218a2e44f485b94239e" + - id: "" + items: + type: object + required: [ id ] + properties: + id: + type: string + globalAliases: + type: array + items: + type: string + localAliases: + type: array + items: + type: object + required: [ alias, accessKeyId ] + properties: + alias: + type: string + accessKeyId: + type: string + + /bucket: + post: + tags: + - Bucket + operationId: "CreateBucket" + summary: "Create a bucket" + description: | + Creates a new bucket, either with a global alias, a local one, or no alias at all. + Technically, you can also specify both `globalAlias` and `localAlias` and that would create two aliases. + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ ] + properties: + globalAlias: + type: string + example: "my_documents" + localAlias: + type: object + properties: + accessKeyId: + type: string + alias: + type: string + allow: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "The payload is not formatted correctly" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + get: + tags: + - Bucket + operationId: "GetBucketInfo" + summary: "Get a bucket" + description: | + Given a bucket identifier (`id`) or a global alias (`globalAlias`), get its information. + It includes its aliases, its web configuration, keys that have some permissions + on it, some statistics (number of objects, size), number of dangling multipart uploads, + and its quotas (if any). + parameters: + - name: id + in: query + description: | + The exact bucket identifier, a 32 bytes hexadecimal string. + + Incompatible with `alias`. + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + - name: globalAlias + in: query + description: | + The exact global alias of one of the existing buckets. + + Incompatible with `id`. + example: "my_documents" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + + delete: + tags: + - Bucket + operationId: "DeleteBucket" + summary: "Delete a bucket" + description: | + Delete a bucket.Deletes a storage bucket. A bucket cannot be deleted if it is not empty. + + **Warning:** this will delete all aliases associated with the bucket! + parameters: + - name: id + in: query + required: true + description: "The exact bucket identifier, a 32 bytes hexadecimal string" + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bucket is not empty" + '404': + description: "Bucket not found" + '204': + description: Bucket has been deleted + + + + put: + tags: + - Bucket + operationId: "UpdateBucket" + summary: "Update a bucket" + description: | + All fields (`websiteAccess` and `quotas`) are optional. + If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. + + In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. + The field `errorDocument` is optional, if no error document is set a generic + error message is displayed when errors happen. Conversely, if `enabled` is + `false`, neither `indexDocument` nor `errorDocument` must be specified. + + In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null` + to remove the quotas. An absent value will be considered the same as a `null`. It is not possible + to change only one of the two quotas. + parameters: + - name: id + in: query + required: true + description: "The exact bucket identifier, a 32 bytes hexadecimal string" + example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87" + schema: + type: string + requestBody: + description: | + Requested changes on the bucket. Both root fields are optionals. + required: true + content: + application/json: + schema: + type: object + required: [ ] + properties: + websiteAccess: + type: object + properties: + enabled: + type: boolean + example: true + indexDocument: + type: string + example: "index.html" + errorDocument: + type: string + example: "error/400.html" + quotas: + type: object + properties: + maxSize: + type: integer + format: int64 + nullable: true + example: 19029801 + maxObjects: + type: integer + format: int64 + nullable: true + example: null + + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your body." + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/allow: + post: + tags: + - Bucket + operationId: "AllowBucketKey" + summary: "Allow key" + description: | + ⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious. + + Allows a key to do read/write/owner operations on a bucket. + + Flags in permissions which have the value true will be activated. Other flags will remain unchanged (ie. they will keep their internal value). + + For example, if you set read to true, the key will be allowed to read the bucket. + If you set it to false, the key will keeps its previous read permission. + If you want to disallow read for the key, check the DenyBucketKey operation. + + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ bucketId, accessKeyId, permissions ] + properties: + bucketId: + type: string + example: "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + permissions: + type: object + required: [ read, write, owner ] + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/deny: + post: + tags: + - Bucket + operationId: "DenyBucketKey" + summary: "Deny key" + description: | + ⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious. + + Denies a key from doing read/write/owner operations on a bucket. + + Flags in permissions which have the value true will be deactivated. Other flags will remain unchanged. + + For example, if you set read to true, the key will be denied from reading. + If you set read to false, the key will keep its previous permissions. + If you want the key to have the reading permission, check the AllowBucketKey operation. + + requestBody: + description: | + Aliases to put on the new bucket + required: true + content: + application/json: + schema: + type: object + required: [ bucketId, accessKeyId, permissions ] + properties: + bucketId: + type: string + example: "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + permissions: + type: object + required: [ read, write, owner ] + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/alias/global: + put: + tags: + - Bucket + operationId: "PutBucketGlobalAlias" + summary: "Add a global alias" + description: | + Add a global alias to the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: alias + in: query + required: true + example: my_documents + schema: + type: string + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + delete: + tags: + - Bucket + operationId: "DeleteBucketGlobalAlias" + summary: "Delete a global alias" + description: | + Delete a global alias from the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: alias + in: query + required: true + schema: + type: string + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + /bucket/alias/local: + put: + tags: + - Bucket + operationId: "PutBucketLocalAlias" + summary: "Add a local alias" + description: | + Add a local alias, bound to specified account, to the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: accessKeyId + in: query + required: true + schema: + type: string + example: GK31c2f218a2e44f485b94239e + - name: alias + in: query + required: true + schema: + type: string + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + + delete: + tags: + - Bucket + operationId: "DeleteBucketLocalAlias" + summary: "Delete a local alias" + description: | + Delete a local alias, bound to specified account, from the target bucket + parameters: + - name: id + in: query + required: true + schema: + type: string + example: e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b + - name: accessKeyId + in: query + schema: + type: string + required: true + example: GK31c2f218a2e44f485b94239e + - name: alias + in: query + schema: + type: string + required: true + example: my_documents + responses: + '500': + description: "The server can not handle your request. Check your connectivity with the rest of the cluster." + '400': + description: "Bad request, check your request body" + '404': + description: "Bucket not found" + '200': + description: Returns exhaustive information about the bucket + content: + application/json: + schema: + $ref: '#/components/schemas/BucketInfo' + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + schemas: + NodeNetworkInfo: + type: object + required: [ addr, isUp, lastSeenSecsAgo, hostname ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + addr: + type: string + example: "10.0.0.11:3901" + isUp: + type: boolean + example: true + lastSeenSecsAgo: + type: integer + nullable: true + example: 9 + hostname: + type: string + example: "node1" + NodeClusterInfo: + type: object + required: [ id, zone, tags ] + properties: + zone: + type: string + example: dc1 + capacity: + type: integer + format: int64 + nullable: true + example: 4 + tags: + type: array + description: | + User defined tags, put whatever makes sense for you, these tags are not interpreted by Garage + example: + - gateway + - fast + items: + type: string + NodeRoleChange: + oneOf: + - $ref: '#/components/schemas/NodeRoleRemove' + - $ref: '#/components/schemas/NodeRoleUpdate' + NodeRoleRemove: + type: object + required: [ id, remove ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + remove: + type: boolean + example: true + NodeRoleUpdate: + type: object + required: [ id, zone, capacity, tags ] + properties: + id: + type: string + example: "6a8e08af2aab1083ebab9b22165ea8b5b9d333b60a39ecd504e85cc1f432c36f" + zone: + type: string + example: "dc1" + capacity: + type: integer + format: int64 + nullable: true + example: 150000000000 + tags: + type: array + items: + type: string + example: + - gateway + - fast + + ClusterLayout: + type: object + required: [ version, roles, stagedRoleChanges ] + properties: + version: + type: integer + example: 12 + roles: + type: array + example: + - id: "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f" + zone: "madrid" + capacity: 300000000000 + tags: + - fast + - amd64 + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + zone: "geneva" + capacity: 700000000000 + tags: + - arm64 + items: + $ref: '#/components/schemas/NodeClusterInfo' + stagedRoleChanges: + type: array + example: + - id: "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b" + zone: "geneva" + capacity: 800000000000 + tags: + - gateway + - id: "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff" + remove: true + items: + $ref: '#/components/schemas/NodeRoleChange' + LayoutVersion: + type: object + required: [ version ] + properties: + version: + type: integer + #format: int64 + example: 13 + + KeyInfo: + type: object + properties: + name: + type: string + example: "test-key" + accessKeyId: + type: string + example: "GK31c2f218a2e44f485b94239e" + secretAccessKey: + type: string + nullable: true + example: "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835" + permissions: + type: object + properties: + createBucket: + type: boolean + example: false + buckets: + type: array + items: + type: object + properties: + id: + type: string + example: "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033" + globalAliases: + type: array + items: + type: string + example: "my-bucket" + localAliases: + type: array + items: + type: string + example: "GK31c2f218a2e44f485b94239e:localname" + permissions: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: false + BucketInfo: + type: object + properties: + id: + type: string + example: afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39 + globalAliases: + type: array + items: + type: string + example: "my_documents" + websiteAccess: + type: boolean + example: true + websiteConfig: + type: object + nullable: true + properties: + indexDocument: + type: string + example: "index.html" + errorDocument: + type: string + example: "error/400.html" + keys: + type: array + items: + $ref: '#/components/schemas/BucketKeyInfo' + objects: + type: integer + format: int64 + example: 14827 + bytes: + type: integer + format: int64 + example: 13189855625 + unfinishedUploads: + type: integer + example: 0 + quotas: + type: object + properties: + maxSize: + nullable: true + type: integer + format: int64 + example: null + maxObjects: + nullable: true + type: integer + format: int64 + example: null + + + BucketKeyInfo: + type: object + properties: + accessKeyId: + type: string + name: + type: string + permissions: + type: object + properties: + read: + type: boolean + example: true + write: + type: boolean + example: true + owner: + type: boolean + example: true + bucketLocalAliases: + type: array + items: + type: string + example: "my_documents" + + +security: + - bearerAuth: [] + +servers: + - description: A local server + url: http://localhost:3903/v1/ diff --git a/doc/book/build/_index.md b/doc/book/build/_index.md index 9bb17086..021045aa 100644 --- a/doc/book/build/_index.md +++ b/doc/book/build/_index.md @@ -1,6 +1,6 @@ +++ title = "Build your own app" -weight = 4 +weight = 40 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/build/golang.md b/doc/book/build/golang.md index a508260e..f3f28a40 100644 --- a/doc/book/build/golang.md +++ b/doc/book/build/golang.md @@ -37,30 +37,84 @@ import ( "context" "fmt" "os" + "strings" garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang" ) func main() { - // Set Host and other parameters + // Initialization configuration := garage.NewConfiguration() configuration.Host = "127.0.0.1:3903" - - - // We can now generate a client client := garage.NewAPIClient(configuration) - - // Authentication is handled through the context pattern ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t") - // Send a request - resp, r, err := client.NodesApi.GetNodes(ctx).Execute() - if err != nil { - fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err) - fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + // Nodes + fmt.Println("--- nodes ---") + nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute() + fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname) + capa := int64(1000000000) + change := []garage.NodeRoleChange{ + garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate { + Id: *nodes.KnownNodes[0].Id, + Zone: "dc1", + Capacity: *garage.NewNullableInt64(&capa), + Tags: []string{ "fast", "amd64" }, + }}, } + staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute() + msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute() + fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured - // Process the response - fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname) + health, _, _ := client.NodesApi.GetHealth(ctx).Execute() + fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions) + + // Key + fmt.Println("\n--- key ---") + key := "openapi-key" + keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute() + defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute() + fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get()) + + id := *keyInfo.AccessKeyId + canCreateBucket := true + updateKeyRequest := *garage.NewUpdateKeyRequest() + updateKeyRequest.SetName("openapi-key-updated") + updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket }) + update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute() + fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name) + + keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute() + fmt.Printf("Keys count: %v\n", len(keyList)) + + // Bucket + fmt.Println("\n--- bucket ---") + global_name := "global-ns-openapi-bucket" + local_name := "local-ns-openapi-bucket" + bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{ + GlobalAlias: &global_name, + LocalAlias: &garage.CreateBucketRequestLocalAlias { + AccessKeyId: keyInfo.AccessKeyId, + Alias: &local_name, + }, + }).Execute() + defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute() + fmt.Printf("Bucket id: %s\n", *bucketInfo.Id) + + updateBucketRequest := *garage.NewUpdateBucketRequest() + website := garage.NewUpdateBucketRequestWebsiteAccess() + website.SetEnabled(true) + website.SetIndexDocument("index.html") + website.SetErrorDocument("errors/4xx.html") + updateBucketRequest.SetWebsiteAccess(*website) + quotas := garage.NewUpdateBucketRequestQuotas() + quotas.SetMaxSize(1000000000) + quotas.SetMaxObjects(999999999) + updateBucketRequest.SetQuotas(*quotas) + updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute() + fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess) + + bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute() + fmt.Printf("Bucket count: %v\n", len(bucketList)) } ``` diff --git a/doc/book/build/javascript.md b/doc/book/build/javascript.md index ff009ffe..a065c595 100644 --- a/doc/book/build/javascript.md +++ b/doc/book/build/javascript.md @@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js. A short example: ```javascript -const garage = require('garage_administration_api_v0garage_v0_8_0'); +const garage = require('garage_administration_api_v1garage_v0_9_0'); -const api = new garage.ApiClient("http://127.0.0.1:3903/v0"); +const api = new garage.ApiClient("http://127.0.0.1:3903/v1"); api.authentications['bearerAuth'].accessToken = "s3cr3t"; const [node, layout, key, bucket] = [ diff --git a/doc/book/build/python.md b/doc/book/build/python.md index 5b797897..612394e1 100644 --- a/doc/book/build/python.md +++ b/doc/book/build/python.md @@ -23,7 +23,7 @@ client = minio.Minio( "GKyourapikey", "abcd[...]1234", # Force the region, this is specific to garage - region="region", + region="garage", ) ``` @@ -80,7 +80,7 @@ from garage_admin_sdk.apis import * from garage_admin_sdk.models import * configuration = garage_admin_sdk.Configuration( - host = "http://localhost:3903/v0", + host = "http://localhost:3903/v1", access_token = "s3cr3t" ) @@ -94,13 +94,14 @@ print(f"running garage {status.garage_version}, node_id {status.node}") # Change layout of this node current = layout.get_layout() -layout.add_layout({ - status.node: NodeClusterInfo( +layout.add_layout([ + NodeRoleChange( + id = status.node, zone = "dc1", - capacity = 1, + capacity = 1000000000, tags = [ "dev" ], ) -}) +]) layout.apply_layout(LayoutVersion( version = current.version + 1 )) diff --git a/doc/book/connect/_index.md b/doc/book/connect/_index.md index 93a2b87e..7d8e686c 100644 --- a/doc/book/connect/_index.md +++ b/doc/book/connect/_index.md @@ -1,6 +1,6 @@ +++ title = "Existing integrations" -weight = 3 +weight = 30 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/connect/apps/index.md b/doc/book/connect/apps/index.md index 4d556ff8..f52d434b 100644 --- a/doc/book/connect/apps/index.md +++ b/doc/book/connect/apps/index.md @@ -11,7 +11,8 @@ In this section, we cover the following web applications: | [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported | | [Mastodon](#mastodon) | ✅ | Natively supported | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | -| [Pixelfed](#pixelfed) | ❓ | Not yet tested | +| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` | +| [Pixelfed](#pixelfed) | ✅ | Natively supported | | [Pleroma](#pleroma) | ❓ | Not yet tested | | [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Funkwhale](#funkwhale) | ❓ | Not yet tested | @@ -36,7 +37,7 @@ Second, we suppose you have created a key and a bucket. As a reminder, you can create a key for your nextcloud instance as follow: ```bash -garage key new --name nextcloud-key +garage key create nextcloud-key ``` Keep the Key ID and the Secret key in a pad, they will be needed later. @@ -68,7 +69,7 @@ $CONFIG = array( 'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com 'port' => 3900, // Put your reverse proxy port or your S3 API port 'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy - 'region' => 'garage', // Garage has only one region named "garage" + 'region' => 'garage', // Garage default region is named "garage", edit according to your cluster config 'use_path_style' => true // Garage supports only path style, must be set to true ], ], @@ -79,6 +80,53 @@ To test your new configuration, just reload your Nextcloud webpage and start sen *External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html) +#### SSE-C encryption (since Garage v1.0) + +Since version 1.0, Garage supports server-side encryption with customer keys +(SSE-C). In this mode, Garage is responsible for encrypting and decrypting +objects, but it does not store the encryption key itself. The encryption key +should be provided by Nextcloud upon each request. This mode of operation is +supported by Nextcloud and it has successfully been tested together with +Garage. + +To enable SSE-C encryption: + +1. Make sure your Garage server is accessible via SSL through a reverse proxy + such as Nginx, and that it is using a valid public certificate (Nextcloud + might be able to connect to an S3 server that is using a self-signed + certificate, but you will lose many hours while trying, so don't). + Configure values for `use_ssl` and `port` accordingly in your `config.php` + file. + +2. Generate an encryption key using the following command: + + ``` + openssl rand -base64 32 + ``` + + Make sure to keep this key **secret**! + +3. Add the encryption key in your `config.php` file as follows: + + + ```php + [ + 'class' => '\\OC\\Files\\ObjectStore\\S3', + 'arguments' => [ + ... + 'sse_c_key' => 'exampleencryptionkeyLbU+5fKYQcVoqnn+RaIOXgo=', + ... + ], + ], + ``` + +Nextcloud will now make Garage encrypt files at rest in the storage bucket. +These files will not be readable by an S3 client that has credentials to the +bucket but doesn't also know the secret encryption key. + + ### External Storage **From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language). @@ -87,7 +135,7 @@ To test your new configuration, just reload your Nextcloud webpage and start sen *Click on the picture to zoom* Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field. -In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen. +In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen. Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared"). @@ -138,15 +186,15 @@ a reasonable trade-off for some instances. Create a key for Peertube: ```bash -garage key new --name peertube-key +garage key create peertube-key ``` Keep the Key ID and the Secret key in a pad, they will be needed later. -We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist). +We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists). ```bash -garage bucket create peertube-video -garage bucket create peertube-playlist +garage bucket create peertube-videos +garage bucket create peertube-playlists ``` Now we allow our key to read and write on these buckets: @@ -190,7 +238,7 @@ object_storage: # Put localhost only if you have a garage instance running on that node endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443 - # Garage supports only one region for now, named garage + # Garage default region is named "garage", edit according to your config region: 'garage' credentials: @@ -205,7 +253,7 @@ object_storage: proxify_private_files: false streaming_playlists: - bucket_name: 'peertube-playlist' + bucket_name: 'peertube-playlists' # Keep it empty for our example prefix: '' @@ -215,7 +263,7 @@ object_storage: # Same settings but for webtorrent videos videos: - bucket_name: 'peertube-video' + bucket_name: 'peertube-videos' prefix: '' # You must fill this field to make Peertube use our reverse proxy/website logic base_url: 'http://peertube-videos.web.garage.localhost' @@ -244,7 +292,7 @@ with average object size ranging from 50 KB to 150 KB. As such, your Garage cluster should be configured appropriately for good performance: - use Garage v0.8.0 or higher with the [LMDB database engine](@documentation/reference-manual/configuration.md#db-engine-since-v0-8-0). - With the default Sled database engine, your database could quickly end up taking tens of GB of disk space. + Older versions of Garage used the Sled database engine which had issues, such as databases quickly ending up taking tens of GB of disk space. - the Garage database should be stored on a SSD ### Creating your bucket @@ -252,7 +300,7 @@ As such, your Garage cluster should be configured appropriately for good perform This is the usual Garage setup: ```bash -garage key new --name mastodon-key +garage key create mastodon-key garage bucket create mastodon-data garage bucket allow mastodon-data --read --write --key mastodon-key ``` @@ -287,6 +335,7 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t ```bash $ RAILS_ENV=production bin/tootctl media remove --days 3 +$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles $ RAILS_ENV=production bin/tootctl media remove-orphans $ RAILS_ENV=production bin/tootctl preview_cards remove --days 15 ``` @@ -305,8 +354,6 @@ Imports: 1.7 KB Settings: 0 Bytes ``` -Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567). - ### Migrating your data Data migration should be done with an efficient S3 client. @@ -378,7 +425,7 @@ Supposing you have a working synapse installation, you can add the module with p Now create a bucket and a key for your matrix instance (note your Key ID and Secret Key somewhere, they will be needed later): ```bash -garage key new --name matrix-key +garage key create matrix-key garage bucket create matrix garage bucket allow matrix --read --write --key matrix-key ``` @@ -394,7 +441,7 @@ media_storage_providers: store_synchronous: True # do we want to wait that the file has been written before returning? config: bucket: matrix # the name of our bucket, we chose matrix earlier - region_name: garage # only "garage" is supported for the region field + region_name: garage # "garage" by default, edit according to your cluster config endpoint_url: http://localhost:3900 # the path to the S3 endpoint access_key_id: "GKxxx" # your Key ID secret_access_key: "xxxx" # your Secret Key @@ -420,7 +467,7 @@ Now we can write a simple script (eg `~/.local/bin/matrix-cache-gc`): ## CONFIGURATION ## AWS_ACCESS_KEY_ID=GKxxx AWS_SECRET_ACCESS_KEY=xxxx -S3_ENDPOINT=http://localhost:3900 +AWS_ENDPOINT_URL=http://localhost:3900 S3_BUCKET=matrix MEDIA_STORE=/var/lib/matrix-synapse/media PG_USER=matrix @@ -441,7 +488,7 @@ EOF s3_media_upload update-db 1d s3_media_upload --no-progress check-deleted $MEDIA_STORE -s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $S3_ENDPOINT +s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $AWS_ENDPOINT_URL ``` This script will list all the medias that were not accessed in the 24 hours according to your database. @@ -474,6 +521,52 @@ And add a new line. For example, to run it every 10 minutes: *External link:* [matrix-media-repo Documentation > S3](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html) +## ejabberd + +ejabberd is an XMPP server implementation which, with the `mod_s3_upload` +module in the [ejabberd-contrib](https://github.com/processone/ejabberd-contrib) +repository, can be integrated to store chat media files in Garage. + +For uploads, this module leverages presigned URLs - this allows XMPP clients to +directly send media to Garage. Receiving clients then retrieve this media +through the [static website](@/documentation/cookbook/exposing-websites.md) +functionality. + +As the data itself is publicly accessible to someone with knowledge of the +object URL - users are recommended to use +[E2EE](@/documentation/cookbook/encryption.md) to protect this data-at-rest +from unauthorized access. + +Install the module with: + +```bash +ejabberdctl module_install mod_s3_upload +``` + +Create the required key and bucket with: + +```bash +garage key new --name ejabberd +garage bucket create objects.xmpp-server.fr +garage bucket allow objects.xmpp-server.fr --read --write --key ejabberd +garage bucket website --allow objects.xmpp-server.fr +``` + +The module can then be configured with: + +``` + mod_s3_upload: + #bucket_url: https://objects.xmpp-server.fr.my-garage-instance.mydomain.tld + bucket_url: https://my-garage-instance.mydomain.tld/objects.xmpp-server.fr + access_key_id: GK... + access_key_secret: ... + region: garage + download_url: https://objects.xmpp-server.fr +``` + +Other configuration options can be found in the +[configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml). + ## Pixelfed [Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem) @@ -539,7 +632,7 @@ secret_key = 'abcdef0123456789...' ``` PICTRS__STORE__TYPE=object_storage -PICTRS__STORE__ENDPOINT=http:/my-garage-instance.mydomain.tld:3900 +PICTRS__STORE__ENDPOINT=http://my-garage-instance.mydomain.tld:3900 PICTRS__STORE__BUCKET_NAME=pictrs-data PICTRS__STORE__REGION=garage PICTRS__STORE__ACCESS_KEY=GK... diff --git a/doc/book/connect/backup.md b/doc/book/connect/backup.md index f51dda30..7e97d777 100644 --- a/doc/book/connect/backup.md +++ b/doc/book/connect/backup.md @@ -54,9 +54,9 @@ how to configure this. Create your key and bucket: ```bash -garage key new my-key -garage bucket create backup -garage bucket allow backup --read --write --key my-key +garage key create my-key +garage bucket create backups +garage bucket allow backups --read --write --key my-key ``` Then register your Key ID and Secret key in your environment: @@ -105,6 +105,7 @@ restic restore 79766175 --target /var/lib/postgresql Restic has way more features than the ones presented here. You can discover all of them by accessing its documentation from the link below. +Files on Android devices can also be backed up with [restic-android](https://github.com/lhns/restic-android). *External links:* [Restic Documentation > Amazon S3](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#amazon-s3) @@ -160,3 +161,49 @@ kopia repository validate-provider You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`... Everything should work out-of-the-box. + +## Plakar + +Create your key and bucket on Garage server: + +```bash +garage key create my-plakar-key +garage bucket create plakar-backups +garage bucket allow plakar-backups --read --write --key my-plakar-key +… +``` + +On Plakar server, add your Garage as a storage location: +```bash +plakar store add garageS3 s3://my-garage.tld/plakar-backups \ +region=garage # Or as you've specified in garage.toml \ +access_key= \ +secret_access_key= +``` + +Then create the repository. +```bash +plakar at @garageS3 create -plaintext # Unencrypted +# or +plakar at @garageS3 create #encrypted +``` + +If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups. + + +After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message): +```bash +plakar at @garageS3 check +``` + +Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage: +```bash +$ plakar at ~/backups sync to @garageS3 +``` + +Or list the S3 storage content: +```bash +$ plakar at @garageS3 ls +``` + +More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/ diff --git a/doc/book/connect/cli.md b/doc/book/connect/cli.md index 9b74bb5d..6529e4b2 100644 --- a/doc/book/connect/cli.md +++ b/doc/book/connect/cli.md @@ -70,16 +70,17 @@ Then a file named `~/.aws/config` and put: ```toml [default] region=garage +endpoint_url=http://127.0.0.1:3900 ``` Now, supposing Garage is listening on `http://127.0.0.1:3900`, you can list your buckets with: ```bash -aws --endpoint-url http://127.0.0.1:3900 s3 ls +aws s3 ls ``` -Passing the `--endpoint-url` parameter to each command is annoying but AWS developers do not provide a corresponding configuration entry. -As a workaround, you can redefine the aws command by editing the file `~/.bashrc`: +If you're using awscli `<1.29.0` or `<2.13.0`, you need to pass `--endpoint-url` to each CLI invocation explicitly. +As a workaround, you can redefine the aws command by editing the file `~/.bashrc` in this case: ``` function aws { command aws --endpoint-url http://127.0.0.1:3900 $@ ; } @@ -202,56 +203,11 @@ See its usage output for other commands available. ## Cyberduck & duck {#cyberduck} Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider. -We wrote the following connection profile for Garage: -```xml - - - - - Protocol - s3 - Vendor - garage - Scheme - https - Description - GarageS3 - Default Hostname - 127.0.0.1 - Default Port - 4443 - Hostname Configurable - - Port Configurable - - Username Configurable - - Username Placeholder - Access Key ID (GK...) - Password Placeholder - Secret Key - Properties - - s3service.disable-dns-buckets=true - - Region - garage - Regions - - garage - - - -``` - -*Note: If your garage instance is configured with vhost access style, you can remove `s3service.disable-dns-buckets=true`.* - -### Instructions for the GUI - -Copy the connection profile, and save it anywhere as `garage.cyberduckprofile`. -Then find this file with your file explorer and double click on it: Cyberduck will open a connection wizard for this profile. -Simply follow the wizard and you should be done! +Within Cyberduck, a +[Garage connection profile](https://docs.cyberduck.io/protocols/s3/garage/) is +available within the `Preferences -> Profiles` section. This can enabled and +then connections to Garage may be configured. ### Instuctions for the CLI @@ -303,7 +259,7 @@ duck --delete garage:/my-files/an-object.txt ## WinSCP (libs3) {#winscp} -*You can find instructions on how to use the GUI in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).* +*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/prise_en_main/winscp/).* How to use `winscp.com`, the CLI interface of WinSCP: diff --git a/doc/book/connect/repositories.md b/doc/book/connect/repositories.md index 4b14bb46..537b02e7 100644 --- a/doc/book/connect/repositories.md +++ b/doc/book/connect/repositories.md @@ -17,13 +17,13 @@ Garage can also help you serve this content. ## Gitea -You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachements. +You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachments. You can configure a different target for each data type (check `[lfs]` and `[attachment]` sections of the Gitea documentation) and you can provide a default one through the `[storage]` section. Let's start by creating a key and a bucket (your key id and secret will be needed later, keep them somewhere): ```bash -garage key new --name gitea-key +garage key create gitea-key garage bucket create gitea garage bucket allow gitea --read --write --key gitea-key ``` @@ -118,7 +118,7 @@ through another support, like a git repository. As a first step, we will need to create a bucket on Garage and enabling website access on it: ```bash -garage key new --name nix-key +garage key create nix-key garage bucket create nix.example.com garage bucket allow nix.example.com --read --write --key nix-key garage bucket website nix.example.com --allow diff --git a/doc/book/cookbook/_index.md b/doc/book/cookbook/_index.md index 07bf6ebf..ff90ad52 100644 --- a/doc/book/cookbook/_index.md +++ b/doc/book/cookbook/_index.md @@ -1,7 +1,7 @@ +++ title="Cookbook" template = "documentation.html" -weight = 2 +weight = 20 sort_by = "weight" +++ @@ -37,7 +37,3 @@ This chapter could also be referred as "Tutorials" or "Best practices". - **[Monitoring Garage](@/documentation/cookbook/monitoring.md)** This page explains the Prometheus metrics available for monitoring the Garage cluster/nodes. - -- **[Recovering from failures](@/documentation/cookbook/recovering.md):** Garage's first selling point is resilience - to hardware failures. This section explains how to recover from such a failure in the - best possible way. diff --git a/doc/book/cookbook/ansible.md b/doc/book/cookbook/ansible.md index 6d624c9c..8b0d2969 100644 --- a/doc/book/cookbook/ansible.md +++ b/doc/book/cookbook/ansible.md @@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below. ## Comparison of Ansible roles -| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | -|------------------------------------|---------------------------------------------|---------------------------------------------------------------| -| **Runtime** | Systemd | Docker | -| **Target OS** | Any Linux | Any Linux | -| **Architecture** | amd64, arm64, i686 | amd64, arm64 | -| **Additional software** | None | Traefik | -| **Automatic node connection** | ❌ | ✅ | -| **Layout management** | ❌ | ✅ | -| **Manage buckets & keys** | ❌ | ✅ (basic) | -| **Allow custom Garage config** | ✅ | ❌ | -| **Facilitate Garage upgrades** | ✅ | ❌ | -| **Multiple instances on one host** | ✅ | ✅ | +| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) | +|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------| +| **Runtime** | Systemd | Docker | Systemd | +| **Target OS** | Any Linux | Any Linux | Any Linux | +| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 | +| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) | +| **Automatic node connection** | ❌ | ✅ | ✅ | +| **Layout management** | ❌ | ✅ | ✅ | +| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ | +| **Allow custom Garage config** | ✅ | ❌ | ❌ | +| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ | +| **Multiple instances on one host** | ✅ | ✅ | ❌ | ## zorun/ansible-role-garage @@ -49,3 +49,15 @@ structured DNS names, etc). As a result, this role makes it easier to start with Garage on Ansible, but is less flexible. + +## eddster2309/ansible-role-garage + +[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/) + +This role is a opinionated but customisable role using the official Garage +static binaries and only requires Systemd. As such it should work on any +Linux based host. It includes all the nesscary configuration to +automatically setup a clustered Garage deployment. Most Garage +configuration options are exposed through Ansible variables so while you +can't provide a custom config you can get very close. It can optionally +installed a HA nginx deployment with Keepalived. diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index 606de2b6..ce6beb7b 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -7,13 +7,27 @@ Garage is also available in binary packages on: ## Alpine Linux +If you use Alpine Linux, you can simply install the +[garage](https://pkgs.alpinelinux.org/packages?name=garage) package from the +Alpine Linux repositories (available since v3.17): + ```bash -apk install garage +apk add garage ``` +The default configuration file is installed to `/etc/garage/garage.toml`. You can run +Garage using: `rc-service garage start`. + +If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start. + +Please note that this package is built without Consul discovery, Kubernetes +discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once +it's stable). + + ## Arch Linux -Garage is available in the [AUR](https://aur.archlinux.org/packages/garage). +Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage). ## FreeBSD diff --git a/doc/book/cookbook/encryption.md b/doc/book/cookbook/encryption.md new file mode 100644 index 00000000..bfbea0ec --- /dev/null +++ b/doc/book/cookbook/encryption.md @@ -0,0 +1,139 @@ ++++ +title = "Encryption" +weight = 50 ++++ + +Encryption is a recurring subject when discussing Garage. +Garage does not handle data encryption by itself, but many things can +already be done with Garage's current feature set and the existing ecosystem. + +This page takes a high level approach to security in general and data encryption +in particular. + + +# Examining your need for encryption + +- Why do you want encryption in Garage? + +- What is your threat model? What are you fearing? + - A stolen HDD? + - A curious administrator? + - A malicious administrator? + - A remote attacker? + - etc. + +- What services do you want to protect with encryption? + - An existing application? Which one? (eg. Nextcloud) + - An application that you are writing + +- Any expertise you may have on the subject + +This page explains what Garage provides, and how you can improve the situation by yourself +by adding encryption at different levels. + +We would be very curious to know your needs and thougs about ideas such as +encryption practices and things like key management, as we want Garage to be a +serious base platform for the developpment of secure, encrypted applications. +Do not hesitate to come talk to us if you have any thoughts or questions on the +subject. + + +# Capabilities provided by Garage + +## Traffic is encrypted between Garage nodes + +RPCs between Garage nodes are encrypted. More specifically, contrary to many +distributed software, it is impossible in Garage to have clear-text RPC. We +use the [kuska handshake](https://github.com/Kuska-ssb/handshake) library which +implements a protocol that has been clearly reviewed, Secure ScuttleButt's +Secret Handshake protocol. This is why setting a `rpc_secret` is mandatory, +and that's also why your nodes have super long identifiers. + +## HTTP API endpoints provided by Garage are in clear text + +Adding TLS support built into Garage is not currently planned. + +## Garage stores data in plain text on the filesystem or encrypted using customer keys (SSE-C) + +For standard S3 API requests, Garage does not encrypt data at rest by itself. +For the most generic at rest encryption of data, we recommend setting up your +storage partitions on encrypted LUKS devices. + +If you are developping your own client software that makes use of S3 storage, +we recommend implementing data encryption directly on the client side and never +transmitting plaintext data to Garage. This makes it easy to use an external +untrusted storage provider if necessary. + +Garage does support [SSE-C +encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html), +an encryption mode of Amazon S3 where data is encrypted at rest using +encryption keys given by the client. The encryption keys are passed to the +server in a header in each request, to encrypt or decrypt data at the moment of +reading or writing. The server discards the key as soon as it has finished +using it for the request. This mode allows the data to be encrypted at rest by +Garage itself, but it requires support in the client software. It is also not +adapted to a model where the server is not trusted or assumed to be +compromised, as the server can easily know the encryption keys. Note however +that when using SSE-C encryption, the only Garage node that knows the +encryption key passed in a given request is the node to which the request is +directed (which can be a gateway node), so it is easy to have untrusted nodes +in the cluster as long as S3 API requests containing SSE-C encryption keys are +not directed to them. + +Implementing automatic data encryption directly in Garage without client-side +management of keys (something like +[SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html)) +could make things simpler for end users that don't want to setup LUKS, but also +raises many more questions, especially around key management: for encryption of +data, where could Garage get the encryption keys from? If we encrypt data but +keep the keys in a plaintext file next to them, it's useless. We probably don't +want to have to manage secrets in Garage as it would be very hard to do in a +secure way. At the time of speaking, there are no plans to implement this in +Garage. + + +# Adding data encryption using external tools + +## Encrypting traffic between a Garage node and your client + +You have multiple options to have encryption between your client and a node: + + - Setup a reverse proxy with TLS / ACME / Let's encrypt + - Setup a Garage gateway locally, and only contact the garage daemon on `localhost` + - Only contact your Garage daemon over a secure, encrypted overlay network such as Wireguard + +## Encrypting data at rest + +Protects against the following threats: + +- Stolen HDD + +Crucially, does not protect againt malicious sysadmins or remote attackers that +might gain access to your servers. + +Methods include full-disk encryption with tools such as LUKS. + +## Encrypting data on the client side + +Protects againt the following threats: + +- A honest-but-curious administrator +- A malicious administrator that tries to corrupt your data +- A remote attacker that can read your server's data + +Implementations are very specific to the various applications. Examples: + +- Matrix: uses the OLM protocol for E2EE of user messages. Media files stored + in Matrix are probably encrypted using symmetric encryption, with a key that is + distributed in the end-to-end encrypted message that contains the link to the object. + +- XMPP: clients normally support either OMEMO / OpenPGP for the E2EE of user + messages. Media files are encrypted per + [XEP-0454](https://xmpp.org/extensions/xep-0454.html). + +- Aerogramme: use the user's password as a key to decrypt data in the user's bucket + +- Cyberduck: comes with support for + [Cryptomator](https://docs.cyberduck.io/cryptomator/) which allows users to + create client-side vaults to encrypt files in before they are uploaded to a + cloud storage endpoint. diff --git a/doc/book/cookbook/exposing-websites.md b/doc/book/cookbook/exposing-websites.md index 5f6a5a28..9382a541 100644 --- a/doc/book/cookbook/exposing-websites.md +++ b/doc/book/cookbook/exposing-websites.md @@ -38,7 +38,7 @@ Our website serving logic is as follow: Now we need to infer the URL of your website through your bucket name. Let assume: - - we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#root_domain)) + - we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#web_root_domain)) - our bucket name is `garagehq.deuxfleurs.fr`. Our bucket will be served if the Host field matches one of these 2 values (the port is ignored): diff --git a/doc/book/cookbook/from-source.md b/doc/book/cookbook/from-source.md index bacf93ab..7105c999 100644 --- a/doc/book/cookbook/from-source.md +++ b/doc/book/cookbook/from-source.md @@ -90,6 +90,6 @@ The following feature flags are available in v0.8.0: | `kubernetes-discovery` | optional | Enable automatic registration and discovery
of cluster nodes through the Kubernetes API | | `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API | | `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry | -| `sled` | *by default* | Enable using Sled to store Garage's metadata | -| `lmdb` | optional | Enable using LMDB to store Garage's metadata | -| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata | +| `syslog` | optional | Enable logging to Syslog | +| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata | +| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata | diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md index dfeb3281..f5bceec8 100644 --- a/doc/book/cookbook/kubernetes.md +++ b/doc/book/cookbook/kubernetes.md @@ -11,7 +11,7 @@ Firstly clone the repository: ```bash git clone https://git.deuxfleurs.fr/Deuxfleurs/garage -cd garage/scripts/helm +cd garage/script/helm ``` Deploy with default options: @@ -26,6 +26,13 @@ Or deploy with custom values: helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml ``` +If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart: + +```bash +kubectl apply -k ../k8s/crd +helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml +``` + After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI: ```bash @@ -86,3 +93,62 @@ helm delete --namespace garage garage ``` Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired. + +## Increase PVC size on running Garage instances + +Since the Garage Helm chart creates the data and meta PVC based on `StatefulSet` templates, increasing the PVC size can be a bit tricky. + +### Confirm the `StorageClass` used for Garage supports volume expansion + +Confirm the storage class used for garage. + +```bash +kubectl -n garage get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE +data-garage-0 Bound pvc-080360c9-8ce3-4acf-8579-1701e57b7f3f 30Gi RWO longhorn-local 77d +data-garage-1 Bound pvc-ab8ba697-6030-4fc7-ab3c-0d6df9e3dbc0 30Gi RWO longhorn-local 5d8h +data-garage-2 Bound pvc-3ab37551-0231-4604-986d-136d0fd950ec 30Gi RWO longhorn-local 5d5h +meta-garage-0 Bound pvc-3b457302-3023-4169-846e-c928c5f2ea65 3Gi RWO longhorn-local 77d +meta-garage-1 Bound pvc-49ace2b9-5c85-42df-9247-51c4cf64b460 3Gi RWO longhorn-local 5d8h +meta-garage-2 Bound pvc-99e2e50f-42b4-4128-ae2f-b52629259723 3Gi RWO longhorn-local 5d5h +``` + +In this case, the storage class is `longhorn-local`. Now, check if `ALLOWVOLUMEEXPANSION` is true for the used `StorageClass`. + +```bash +kubectl get storageclasses.storage.k8s.io longhorn-local +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +longhorn-local driver.longhorn.io Delete Immediate true 103d +``` + +If your `StorageClass` does not support volume expansion, double check if you can enable it. Otherwise, your only real option is to spin up a new Garage cluster with increased size and migrate all data over. + +If your `StorageClass` supports expansion, you are free to continue. + +### Increase the size of the PVCs + +Increase the size of all PVCs to your desired size. + +```bash +kubectl -n garage edit pvc data-garage-0 +kubectl -n garage edit pvc data-garage-1 +kubectl -n garage edit pvc data-garage-2 +kubectl -n garage edit pvc meta-garage-0 +kubectl -n garage edit pvc meta-garage-1 +kubectl -n garage edit pvc meta-garage-2 +``` + +### Increase the size of the `StatefulSet` PVC template + +This is an optional step, but if not done, future instances of Garage will be created with the original size from the template. + +```bash +kubectl -n garage delete sts --cascade=orphan garage +statefulset.apps "garage" deleted +``` + +This will remove the Garage `StatefulSet` but leave the pods running. It may seem destructive but needs to be done this way since edits to the size of PVC templates are prohibited. + +### Redeploy the `StatefulSet` + +Now the size of future PVCs can be increased, and the Garage Helm chart can be upgraded. The new `StatefulSet` should take ownership of the orphaned pods again. diff --git a/doc/book/cookbook/monitoring.md b/doc/book/cookbook/monitoring.md index 8313daa9..d643a9ca 100644 --- a/doc/book/cookbook/monitoring.md +++ b/doc/book/cookbook/monitoring.md @@ -18,7 +18,7 @@ api_bind_addr = "0.0.0.0:3903" ``` This will allow anyone to scrape Prometheus metrics by fetching -`http://localhost:3093/metrics`. If you want to restrict access +`http://localhost:3903/metrics`. If you want to restrict access to the exported metrics, set the `metrics_token` configuration value to a bearer token to be used when fetching the metrics endpoint. @@ -49,9 +49,5 @@ add the following lines in your Prometheus scrape config: To visualize the scraped data in Grafana, you can either import our [Grafana dashboard for Garage](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/branch/main/script/telemetry/grafana-garage-dashboard-prometheus.json) or make your own. -We detail below the list of exposed metrics and their meaning. - -## List of exported metrics - -See our [dedicated page](@/documentation/reference-manual/monitoring.md) in the Reference manual section. +The list of exported metrics is available on our [dedicated page](@/documentation/reference-manual/monitoring.md) in the Reference manual section. diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 9be1ba44..b9927c06 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -19,14 +19,15 @@ To run a real-world deployment, make sure the following conditions are met: - You have at least three machines with sufficient storage space available. -- Each machine has a public IP address which is reachable by other machines. It - is highly recommended that you use IPv6 for this end-to-end connectivity. If - IPv6 is not available, then using a mesh VPN such as +- Each machine has an IP address which makes it directly reachable by all other machines. + In many cases, nodes will be behind a NAT and will not each have a public + IPv4 addresses. In this case, is recommended that you use IPv6 for this + end-to-end connectivity if it is available. Otherwise, using a mesh VPN such as [Nebula](https://github.com/slackhq/nebula) or [Yggdrasil](https://yggdrasil-network.github.io/) are approaches to consider in addition to building out your own VPN tunneling. -- This guide will assume you are using Docker containers to deploy Garage on each node. +- This guide will assume you are using Docker containers to deploy Garage on each node. Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md). You can also use an orchestrator such as Nomad or Kubernetes to automatically manage Docker containers on a fleet of nodes. @@ -42,7 +43,7 @@ For our example, we will suppose the following infrastructure with IPv6 connecti | Brussels | Mars | fc00:F::1 | 1.5 TB | Note that Garage will **always** store the three copies of your data on nodes at different -locations. This means that in the case of this small example, the available capacity +locations. This means that in the case of this small example, the usable capacity of the cluster is in fact only 1.5 TB, because nodes in Brussels can't store more than that. This also means that nodes in Paris and London will be under-utilized. To make better use of the available hardware, you should ensure that the capacity @@ -52,9 +53,9 @@ to store 2 TB of data in total. ### Best practices -- If you have fast dedicated networking between all your nodes, and are planing to store - very large files, bump the `block_size` configuration parameter to 10 MB - (`block_size = 10485760`). +- If you have reasonably fast networking between all your nodes, and are planing to store + mostly large files, bump the `block_size` configuration parameter to 10 MB + (`block_size = "10M"`). - Garage stores its files in two locations: it uses a metadata directory to store frequently-accessed small metadata items, and a data directory to store data blocks of uploaded objects. @@ -67,36 +68,42 @@ to store 2 TB of data in total. EXT4 is not recommended as it has more strict limitations on the number of inodes, which might cause issues with Garage when large numbers of objects are stored. -- If you only have an HDD and no SSD, it's fine to put your metadata alongside the data - on the same drive. Having lots of RAM for your kernel to cache the metadata will - help a lot with performance. Make sure to use the LMDB database engine, - instead of Sled, which suffers from quite bad performance degradation on HDDs. - Sled is still the default for legacy reasons, but is not recommended anymore. +- Servers with multiple HDDs are supported natively by Garage without resorting + to RAID, see [our dedicated documentation page](@/documentation/operations/multi-hdd.md). - For the metadata storage, Garage does not do checksumming and integrity - verification on its own. If you are afraid of bitrot/data corruption, - put your metadata directory on a BTRFS partition. Otherwise, just use regular - EXT4 or XFS. + verification on its own, so it is better to use a robust filesystem such as + BTRFS or ZFS. Users have reported that when using the LMDB database engine + (the default), database files have a tendency of becoming corrupted after an + unclean shutdown (e.g. a power outage), so you should take regular snapshots + to be able to recover from such a situation. This can be done using Garage's + built-in automatic snapshotting (since v0.9.4), or by using filesystem level + snapshots. If you cannot do so, you might want to switch to Sqlite which is + more robust. -- Having a single server with several storage drives is currently not very well - supported in Garage ([#218](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/218)). - For an easy setup, just put all your drives in a RAID0 or a ZFS RAIDZ array. - If you're adventurous, you can try to format each of your disk as - a separate XFS partition, and then run one `garage` daemon per disk drive, - or use something like [`mergerfs`](https://github.com/trapexit/mergerfs) to merge - all your disks in a single union filesystem that spreads load over them. +- LMDB is the fastest and most tested database engine, but it has the following + weaknesses: 1/ data files are not architecture-independent, you cannot simply + move a Garage metadata directory between nodes running different architectures, + and 2/ LMDB is not suited for 32-bit platforms. Sqlite is a viable alternative + if any of these are of concern. + +- If you only have an HDD and no SSD, it's fine to put your metadata alongside + the data on the same drive, but then consider your filesystem choice wisely + (see above). Having lots of RAM for your kernel to cache the metadata will + help a lot with performance. The default LMDB database engine is the most + tested and has good performance. ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v0.8.0`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v0.8.0` but it's up to you +We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v0.8.0 +sudo docker pull dxflrs/garage:v1.3.0 ``` ## Deploying and configuring Garage @@ -119,8 +126,9 @@ A valid `/etc/garage.toml` for our cluster would look as follows: metadata_dir = "/var/lib/garage/meta" data_dir = "/var/lib/garage/data" db_engine = "lmdb" +metadata_auto_snapshot_interval = "6h" -replication_mode = "3" +replication_factor = 3 compression_level = 2 @@ -144,6 +152,8 @@ Check the following for your configuration files: - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. This parameter is optional but recommended: if your nodes have trouble communicating with one another, consider adding it. + Alternatively, you can also set `rpc_public_addr_subnet`, which can filter + the addresses announced to other peers to a specific subnet. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. You can generate such a key with `openssl rand -hex 32`. @@ -161,12 +171,13 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v0.8.0 + dxflrs/garage:v1.3.0 ``` -It should be restarted automatically at each reboot. -Please note that we use host networking as otherwise Docker containers -can not communicate with IPv6. +With this command line, Garage should be started automatically at each boot. +Please note that we use host networking as otherwise the network indirection +added by Docker would prevent Garage nodes from communicating with one another +(especially if using IPv6). If you want to use `docker-compose`, you may use the following `docker-compose.yml` file as a reference: @@ -174,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v0.8.0 + image: dxflrs/garage:v1.3.0 network_mode: "host" restart: unless-stopped volumes: @@ -183,12 +194,14 @@ services: - /var/lib/garage/data:/var/lib/garage/data ``` -Upgrading between Garage versions should be supported transparently, -but please check the relase notes before doing so! -To upgrade, simply stop and remove this container and -start again the command with a new version of Garage. +If you wish to upgrade your cluster, make sure to read the corresponding +[documentation page](@/documentation/operations/upgrading.md) first, as well as +the documentation relevant to your version of Garage in the case of major +upgrades. With the containerized setup proposed here, the upgrade process +will require stopping and removing the existing container, and re-creating it +with the upgraded version. -## Controling the daemon +## Controlling the daemon The `garage` binary has two purposes: - it acts as a daemon when launched with `garage server` @@ -197,6 +210,12 @@ The `garage` binary has two purposes: Ensure an appropriate `garage` binary (the same version as your Docker image) is available in your path. If your configuration file is at `/etc/garage.toml`, the `garage` binary should work with no further change. +You can also use an alias as follows to use the Garage binary inside your docker container: + +```bash +alias garage="docker exec -ti /garage" +``` + You can test your `garage` CLI utility by running a simple command such as: ```bash @@ -240,7 +259,7 @@ You can then instruct nodes to connect to one another as follows: Venus$ garage node connect 563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901 ``` -You don't nead to instruct all node to connect to all other nodes: +You don't need to instruct all node to connect to all other nodes: nodes will discover one another transitively. Now if your run `garage status` on any node, you should have an output that looks as follows: @@ -264,12 +283,12 @@ of a role that is assigned to each active cluster node. For our example, we will suppose we have the following infrastructure (Capacity, Identifier and Zone are specific values to Garage described in the following): -| Location | Name | Disk Space | `Capacity` | `Identifier` | `Zone` | -|----------|---------|------------|------------|--------------|--------------| -| Paris | Mercury | 1 TB | `10` | `563e` | `par1` | -| Paris | Venus | 2 TB | `20` | `86f0` | `par1` | -| London | Earth | 2 TB | `20` | `6814` | `lon1` | -| Brussels | Mars | 1.5 TB | `15` | `212f` | `bru1` | +| Location | Name | Disk Space | Identifier | Zone (`-z`) | Capacity (`-c`) | +|----------|---------|------------|------------|-------------|-----------------| +| Paris | Mercury | 1 TB | `563e` | `par1` | `1T` | +| Paris | Venus | 2 TB | `86f0` | `par1` | `2T` | +| London | Earth | 2 TB | `6814` | `lon1` | `2T` | +| Brussels | Mars | 1.5 TB | `212f` | `bru1` | `1.5T` | #### Node identifiers @@ -291,6 +310,8 @@ garage status It will display the IP address associated with each node; from the IP address you will be able to recognize the node. +We will now use the `garage layout assign` command to configure the correct parameters for each node. + #### Zones Zones are simply a user-chosen identifier that identify a group of server that are grouped together logically. @@ -300,29 +321,29 @@ In most cases, a zone will correspond to a geographical location (i.e. a datacen Behind the scene, Garage will use zone definition to try to store the same data on different zones, in order to provide high availability despite failure of a zone. +Zones are passed to Garage using the `-z` flag of `garage layout assign` (see below). + #### Capacity -Garage reasons on an abstract metric about disk storage that is named the *capacity* of a node. -The capacity configured in Garage must be proportional to the disk space dedicated to the node. +Garage needs to know the storage capacity (disk space) it can/should use on +each node, to be able to correctly balance data. -Capacity values must be **integers** but can be given any signification. -Here we chose that 1 unit of capacity = 100 GB. +Capacity values are expressed in bytes and are passed to Garage using the `-c` flag of `garage layout assign` (see below). -Note that the amount of data stored by Garage on each server may not be strictly proportional to -its capacity value, as Garage will priorize having 3 copies of data in different zones, -even if this means that capacities will not be strictly respected. For example in our above examples, -nodes Earth and Mars will always store a copy of everything each, and the third copy will -have 66% chance of being stored by Venus and 33% chance of being stored by Mercury. +#### Tags + +You can add additional tags to nodes using the `-t` flag of `garage layout assign` (see below). +Tags have no specific meaning for Garage and can be used at your convenience. #### Injecting the topology Given the information above, we will configure our cluster as follow: ```bash -garage layout assign 563e -z par1 -c 10 -t mercury -garage layout assign 86f0 -z par1 -c 20 -t venus -garage layout assign 6814 -z lon1 -c 20 -t earth -garage layout assign 212f -z bru1 -c 15 -t mars +garage layout assign 563e -z par1 -c 1T -t mercury +garage layout assign 86f0 -z par1 -c 2T -t venus +garage layout assign 6814 -z lon1 -c 2T -t earth +garage layout assign 212f -z bru1 -c 1.5T -t mars ``` At this point, the changes in the cluster layout have not yet been applied. @@ -332,6 +353,7 @@ To show the new layout that will be applied, call: garage layout show ``` +Make sure to read carefully the output of `garage layout show`. Once you are satisfied with your new layout, apply it with: ```bash @@ -339,7 +361,7 @@ garage layout apply ``` **WARNING:** if you want to use the layout modification commands in a script, -make sure to read [this page](@/documentation/reference-manual/layout.md) first. +make sure to read [this page](@/documentation/operations/layout.md) first. ## Using your Garage cluster @@ -349,5 +371,5 @@ and is covered in the [quick start guide](@/documentation/quick-start/_index.md) Remember also that the CLI is self-documented thanks to the `--help` flag and the `help` subcommand (e.g. `garage help`, `garage key --help`). -Configuring S3-compatible applicatiosn to interact with Garage +Configuring S3-compatible applications to interact with Garage is covered in the [Integrations](@/documentation/connect/_index.md) section. diff --git a/doc/book/cookbook/reverse-proxy.md b/doc/book/cookbook/reverse-proxy.md index 9c833ad0..bdc1c549 100644 --- a/doc/book/cookbook/reverse-proxy.md +++ b/doc/book/cookbook/reverse-proxy.md @@ -378,6 +378,47 @@ admin.garage.tld { But at the same time, the `reverse_proxy` is very flexible. For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc. +### Caching + +Caddy can compiled with a +[cache plugin](https://github.com/caddyserver/cache-handler) which can be used +to provide a hot-cache at the webserver-level for static websites hosted by +Garage. + +This can be configured as follows: + +```caddy +# Caddy global configuration section +{ + # Bare minimum configuration to enable cache. + order cache before rewrite + + cache + + #cache + # allowed_http_verbs GET + # default_cache_control public + # ttl 8h + #} +} + +# Site specific section +https:// { + cache + + #cache { + # timeout { + # backend 30s + # } + #} + + reverse_proxy ... +} +``` + +Caching is a complicated subject, and the reader is encouraged to study the +available options provided by the plugin. + ### On-demand TLS Caddy supports a technique called @@ -428,3 +469,35 @@ https:// { reverse_proxy localhost:3902 192.168.1.2:3902 example.tld:3902 } ``` + +More information on how this endpoint is implemented in Garage is available +in the [Admin API Reference](@/documentation/reference-manual/admin-api.md) page. + +### Fileserver browser + +Caddy's built-in +[file_server](https://caddyserver.com/docs/caddyfile/directives/file_server) +browser functionality can be extended with the +[caddy-fs-s3](https://github.com/sagikazarmark/caddy-fs-s3) module. + +This can be configured to use Garage as a backend with the following +configuration: + +```caddy +browse.garage.tld { + file_server { + fs s3 { + bucket test-bucket + region garage + + endpoint https://s3.garage.tld + use_path_style + } + + browse + } +} +``` + +Caddy must also be configured with the required `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` environment variables to access the bucket. diff --git a/doc/book/cookbook/systemd.md b/doc/book/cookbook/systemd.md index b271010b..ebff8c15 100644 --- a/doc/book/cookbook/systemd.md +++ b/doc/book/cookbook/systemd.md @@ -28,12 +28,26 @@ StateDirectory=garage DynamicUser=true ProtectHome=true NoNewPrivileges=true +LimitNOFILE=42000 [Install] WantedBy=multi-user.target ``` -*A note on hardening: garage will be run as a non privileged user, its user id is dynamically allocated by systemd. It cannot access (read or write) home folders (/home, /root and /run/user), the rest of the filesystem can only be read but not written, only the path seen as /var/lib/garage is writable as seen by the service (mapped to /var/lib/private/garage on your host). Additionnaly, the process can not gain new privileges over time.* +**A note on hardening:** Garage will be run as a non privileged user, its user +id is dynamically allocated by systemd (set with `DynamicUser=true`). It cannot +access (read or write) home folders (`/home`, `/root` and `/run/user`), the +rest of the filesystem can only be read but not written, only the path seen as +`/var/lib/garage` is writable as seen by the service. Additionnaly, the process +can not gain new privileges over time. + +For this to work correctly, your `garage.toml` must be set with +`metadata_dir=/var/lib/garage/meta` and `data_dir=/var/lib/garage/data`. This +is mandatory to use the DynamicUser hardening feature of systemd, which +autocreates these directories as virtual mapping. If the directory +`/var/lib/garage` already exists before starting the server for the first time, +the systemd service might not start correctly. Note that in your host +filesystem, Garage data will be held in `/var/lib/private/garage`. To start the service then automatically enable it at boot: diff --git a/doc/book/design/_index.md b/doc/book/design/_index.md index 50933139..5881ab8f 100644 --- a/doc/book/design/_index.md +++ b/doc/book/design/_index.md @@ -1,6 +1,6 @@ +++ title = "Design" -weight = 6 +weight = 70 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/design/goals.md b/doc/book/design/goals.md index 4e390ba6..efa3cd33 100644 --- a/doc/book/design/goals.md +++ b/doc/book/design/goals.md @@ -42,15 +42,28 @@ locations. They use Garage themselves for the following tasks: - As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider) -- To store personal data and shared documents through [Bagage](https://git.deuxfleurs.fr/Deuxfleurs/bagage), a homegrown WebDav-to-S3 proxy - -- In the Drone continuous integration platform to store task logs - - As a Nix binary cache -- As a backup target using `rclone` +- To store personal data and shared documents through [Bagage](https://git.deuxfleurs.fr/Deuxfleurs/bagage), a homegrown WebDav-to-S3 and SFTP-to-S3 proxy + +- As a backup target using `rclone` and `restic` The Deuxfleurs Garage cluster is a multi-site cluster currently composed of -4 nodes in 2 physical locations. In the future it will be expanded to at -least 3 physical locations to fully exploit Garage's potential for high -availability. +9 nodes in 3 physical locations. + +### Triplebit + +[Triplebit](https://www.triplebit.org) is a non-profit hosting provider and +ISP focused on improving access to privacy-related services. They use +Garage themselves for the following tasks: + +- Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites + +- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/) + +- As a PeerTube storage backend for [neat.tube](https://neat.tube/) + +- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider) + +Triplebit's Garage cluster is a multi-site cluster currently composed of +10 nodes in 3 physical locations. diff --git a/doc/book/design/internals.md b/doc/book/design/internals.md index 777e017d..8e3c214e 100644 --- a/doc/book/design/internals.md +++ b/doc/book/design/internals.md @@ -61,7 +61,7 @@ Garage prioritizes which nodes to query according to a few criteria: For further reading on the cluster structure look at the [gateway](@/documentation/cookbook/gateways.md) -and [cluster layout management](@/documentation/reference-manual/layout.md) pages. +and [cluster layout management](@/documentation/operations/layout.md) pages. ## Garbage collection @@ -97,7 +97,7 @@ delete a tombstone, the following condition has to be met: superseeded by the tombstone. This ensures that deleting the tombstone is safe and that no deleted value will come back in the system. -Garage makes use of Sled's atomic operations (such as compare-and-swap and +Garage uses atomic database operations (such as compare-and-swap and transactions) to ensure that only tombstones that have been correctly propagated to other nodes are ever deleted from the local entry tree. diff --git a/doc/book/design/related-work.md b/doc/book/design/related-work.md index 6c1a6b12..84e66c4e 100644 --- a/doc/book/design/related-work.md +++ b/doc/book/design/related-work.md @@ -67,7 +67,7 @@ Pithos has been abandonned and should probably not used yet, in the following we Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too). From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment. They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it. -We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. +We considered their v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. **[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):** *Not written yet* diff --git a/doc/book/development/_index.md b/doc/book/development/_index.md index 8e730bf6..2b2af0cc 100644 --- a/doc/book/development/_index.md +++ b/doc/book/development/_index.md @@ -1,6 +1,6 @@ +++ title = "Development" -weight = 7 +weight = 80 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/development/devenv.md b/doc/book/development/devenv.md index 8d7d2e95..518fd232 100644 --- a/doc/book/development/devenv.md +++ b/doc/book/development/devenv.md @@ -25,7 +25,7 @@ git clone https://git.deuxfleurs.fr/Deuxfleurs/garage cd garage ``` -*Optionnaly, you can use our nix.conf file to speed up compilations:* +*Optionally, you can use our nix.conf file to speed up compilations:* ```bash sudo mkdir -p /etc/nix @@ -36,7 +36,7 @@ sudo killall nix-daemon Now you can enter our nix-shell, all the required packages will be downloaded but they will not pollute your environment outside of the shell: ```bash -nix-shell +nix-shell -A devShell ``` You can use the traditional Rust development workflow: @@ -65,8 +65,8 @@ nix-build -j $(nproc) --max-jobs auto ``` Our build has multiple parameters you might want to set: - - `release` build with release optimisations instead of debug - - `target allows` for cross compilation + - `release` to build with release optimisations instead of debug + - `target` allows for cross compilation - `compileMode` can be set to test or bench to build a unit test runner - `git_version` to inject the hash to display when running `garage stats` @@ -80,7 +80,7 @@ nix-build \ --git_version $(git rev-parse HEAD) ``` -*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.* +*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.* If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`: diff --git a/doc/book/development/miscellaneous-notes.md b/doc/book/development/miscellaneous-notes.md index f0083ae5..a421943f 100644 --- a/doc/book/development/miscellaneous-notes.md +++ b/doc/book/development/miscellaneous-notes.md @@ -81,12 +81,9 @@ Our cache will be checked. - http://www.lpenz.org/articles/nixchannel/index.html -## Drone +## Woodpecker -Do not try to set a build as trusted from the interface or the CLI tool, -your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`). - -Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. +Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. ## Building Docker containers @@ -99,3 +96,4 @@ We were: - Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json. Finally we chose to build kaniko through nix and use it in a `nix-shell`. +We then switched to using kaniko from nixpkgs when it was packaged. diff --git a/doc/book/development/release-process.md b/doc/book/development/release-process.md index 3fed4add..0c6701c0 100644 --- a/doc/book/development/release-process.md +++ b/doc/book/development/release-process.md @@ -42,7 +42,7 @@ and the docker containers on Docker Hub. ## Automation -We automated our release process with Nix and Drone to make it more reliable. +We automated our release process with Nix and Woodpecker to make it more reliable. Here we describe how we have done in case you want to debug or improve it. ### Caching build steps @@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example: nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' result ``` -*Note that you need the signing key. In our case, it is stored as a secret in Drone.* +*The signing key possessed by the Garage maintainers is required to update the Nix cache.* -The previous command will only send the built packet and not its dependencies. -To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol. - -Instead, you can use the following commands to list all the runtime dependencies: +The previous command will only send the built package and not its dependencies. +In the case of our CI pipeline, we want to cache all intermediate build steps +as well. This can be done using this quite involved command (here as an example +for the `pkgs.amd64.relase` package): ```bash -nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR result/) +nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/nix-signing-key.sec' \ + $(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') ``` -*We could also write this expression with xargs but this tool is not available in our container.* +This command will simultaneously build all of the required Nix paths (using at +most 8 parallel Nix builder jobs) and send the resulting objects to the cache. -But in certain cases, we want to cache compile time dependencies also. -For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own. -We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it. - -This time, the command is a bit more involved: - -```bash -nix copy --to \ - 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate)) -``` - -This is the command we use in our CI as we expect the final binary to change, so we mainly focus on -caching our development dependencies. - -*Currently there is no automatic garbage collection of the cache: we should monitor its growth. -Hopefully, we can erase it totally without breaking any build, the next build will only be slower.* - -In practise, we concluded that we do not want to cache all the compilation dependencies. -Instead, we want to cache the toolchain we use to build Garage each time we change it. -So we removed from Drone any automatic update of the cache and instead handle them manually with: +This can be run for all the Garage packages we build using the following command: ``` source ~/.awsrc -nix-shell --run 'refresh_toolchain' +nix-shell --attr cache --run 'refresh_cache' ``` -Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache. - -To erase the cache: +We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.* +This means we should also monitor the cache's size; if it ever becomes too big we can erase it with: ``` mc rm --recursive --force 'garage/nix/' @@ -157,9 +136,9 @@ nix-shell --run refresh_index If you want to compile for different architectures, you will need to repeat all these commands for each architecture. -**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone** +**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.** -### Drone +### Drone (obsolete) Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr). You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it. diff --git a/doc/book/operations/_index.md b/doc/book/operations/_index.md new file mode 100644 index 00000000..16d0e9d5 --- /dev/null +++ b/doc/book/operations/_index.md @@ -0,0 +1,23 @@ ++++ +title = "Operations & Maintenance" +weight = 50 +sort_by = "weight" +template = "documentation.html" ++++ + +This section contains a number of important information on how to best operate a Garage cluster, +to ensure integrity and availability of your data: + +- **[Upgrading Garage](@/documentation/operations/upgrading.md):** General instructions on how to + upgrade your cluster from one version to the next. Instructions specific for each version upgrade + can bef ound in the [working documents](@/documentation/working-documents/_index.md) section. + +- **[Layout management](@/documentation/operations/layout.md):** Best practices for using the `garage layout` + commands when adding or removing nodes from your cluster. + +- **[Durability and repairs](@/documentation/operations/durability-repairs.md):** How to check for small things + that might be going wrong, and how to recover from such failures. + +- **[Recovering from failures](@/documentation/operations/recovering.md):** Garage's first selling point is resilience + to hardware failures. This section explains how to recover from such a failure in the + best possible way. diff --git a/doc/book/operations/durability-repairs.md b/doc/book/operations/durability-repairs.md new file mode 100644 index 00000000..fdf163e2 --- /dev/null +++ b/doc/book/operations/durability-repairs.md @@ -0,0 +1,147 @@ ++++ +title = "Durability & Repairs" +weight = 30 ++++ + +To ensure the best durability of your data and to fix any inconsistencies that may +pop up in a distributed system, Garage provides a series of repair operations. +This guide will explain the meaning of each of them and when they should be applied. + + +# General syntax of repair operations + +Repair operations described below are of the form `garage repair `. +These repairs will not launch without the `--yes` flag, which should +be added as follows: `garage repair --yes `. +By default these repair procedures will only run on the Garage node your CLI is +connecting to. To run on all nodes, add the `-a` flag as follows: +`garage repair -a --yes `. + +# Data block operations + +## Data store scrub {#scrub} + +Scrubbing the data store means examining each individual data block to check that +their content is correct, by verifying their hash. Any block found to be corrupted +(e.g. by bitrot or by an accidental manipulation of the datastore) will be +restored from another node that holds a valid copy. + +Scrubs are automatically scheduled by Garage to run every 25-35 days (the +actual time is randomized to spread load across nodes). The next scheduled run +can be viewed with `garage worker get`. + +A scrub can also be launched manually using `garage repair scrub start`. + +To view the status of an ongoing scrub, first find the task ID of the scrub worker +using `garage worker list`. Then, run `garage worker info ` to +view detailed runtime statistics of the scrub. To gather cluster-wide information, +this command has to be run on each individual node. + +A scrub is a very disk-intensive operation that might slow down your cluster. +You may pause an ongoing scrub using `garage repair scrub pause`, but note that +the scrub will resume automatically 24 hours later as Garage will not let your +cluster run without a regular scrub. If the scrub procedure is too intensive +for your servers and is slowing down your workload, the recommended solution +is to increase the "scrub tranquility" using `garage repair scrub set-tranquility`. +A higher tranquility value will make Garage take longer pauses between two block +verifications. Of course, scrubbing the entire data store will also take longer. + +## Block check and resync + +In some cases, nodes hold a reference to a block but do not actually have the block +stored on disk. Conversely, they may also have on-disk blocks that are not referenced +any more. To fix both cases, a block repair may be run with `garage repair blocks`. +This will scan the entire block reference counter table to check that the blocks +exist on disk, and will scan the entire disk store to check that stored blocks +are referenced. + +It is recommended to run this procedure when changing your cluster layout, +after the metadata tables have finished synchronizing between nodes +(usually a few hours after `garage layout apply`). + +## Inspecting lost blocks + +In extremely rare situations, data blocks may be unavailable from the entire cluster. +This means that even using `garage repair blocks`, some nodes may be unable +to fetch data blocks for which they hold a reference. + +These errors are stored on each node in a list of "block resync errors", i.e. +blocks for which the last resync operation failed. +This list can be inspected using `garage block list-errors`. +These errors usually fall into one of the following categories: + +1. a block is still referenced but the object was deleted, this is a case + of metadata reference inconsistency (see below for the fix) +2. a block is referenced by a non-deleted object, but could not be fetched due + to a transient error such as a network failure +3. a block is referenced by a non-deleted object, but could not be fetched due + to a permanent error such as there not being any valid copy of the block on the + entire cluster + +To help make the difference between cases 1 and cases 2 and 3, you may use the +`garage block info` command to see which objects hold a reference to each block. + +In the second case (transient errors), Garage will try to fetch the block again +after a certain time, so the error should disappear naturally. You can also +request Garage to try to fetch the block immediately using `garage block retry-now` +if you have fixed the transient issue. + +If you are confident that you are in the third scenario and that your data block +is definitely lost, then there is no other choice than to declare your S3 objects +as unrecoverable, and to delete them properly from the data store. This can be done +using the `garage block purge` command. + +## Rebalancing data directories + +In [multi-HDD setups](@/documentation/operations/multi-hdd.md), to ensure that +data blocks are well balanced between storage locations, you may run a +rebalance operation using `garage repair rebalance`. This is useful when +adding storage locations or when capacities of the storage locations have been +changed. Once this is finished, Garage will know for each block of a single +possible location where it can be, which can increase access speed. This +operation will also move out all data from locations marked as read-only. + + +# Metadata operations + +## Metadata snapshotting + +It is good practice to setup automatic snapshotting of your metadata database +file, to recover from situations where it becomes corrupted on disk. This can +be done at the filesystem level if you are using ZFS or BTRFS. + +Since Garage v0.9.4, Garage is able to take snapshots of the metadata database +itself. This basically amounts to copying the database file, except that it can +be run live while Garage is running without the risk of corruption or +inconsistencies. This can be setup to run automatically on a schedule using +[`metadata_auto_snapshot_interval`](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval). +A snapshot can also be triggered manually using the `garage meta snapshot` +command. Note that taking a snapshot using this method is very intensive as it +requires making a full copy of the database file, so you might prefer using +filesystem-level snapshots if possible. To recover a corrupted node from such a +snapshot, read the instructions +[here](@/documentation/operations/recovering.md#corrupted_meta). + +## Metadata table resync + +Garage automatically resyncs all entries stored in the metadata tables every hour, +to ensure that all nodes have the most up-to-date version of all the information +they should be holding. +The resync procedure is based on a Merkle tree that allows to efficiently find +differences between nodes. + +In some special cases, e.g. before an upgrade, you might want to run a table +resync manually. This can be done using `garage repair tables`. + +## Metadata table reference fixes + +In some very rare cases where nodes are unavailable, some references between objects +are broken. For instance, if an object is deleted, the underlying versions or data +blocks may still be held by Garage. If you suspect that such corruption has occurred +in your cluster, you can run one of the following repair procedures: + +- `garage repair versions`: checks that all versions belong to a non-deleted object, and purges any orphan version + +- `garage repair block-refs`: checks that all block references belong to a non-deleted object version, and purges any orphan block reference (this will then allow the blocks to be garbage-collected) + +- `garage repair block-rc`: checks that the reference counters for blocks are in sync with the actual number of non-deleted entries in the block reference table diff --git a/doc/book/operations/layout.md b/doc/book/operations/layout.md new file mode 100644 index 00000000..667e89d2 --- /dev/null +++ b/doc/book/operations/layout.md @@ -0,0 +1,274 @@ ++++ +title = "Cluster layout management" +weight = 20 ++++ + +The cluster layout in Garage is a table that assigns to each node a role in +the cluster. The role of a node in Garage can either be a storage node with +a certain capacity, or a gateway node that does not store data and is only +used as an API entry point for faster cluster access. +An introduction to building cluster layouts can be found in the [production deployment](@/documentation/cookbook/real-world.md) page. + +In Garage, all of the data that can be stored in a given cluster is divided +into slices which we call *partitions*. Each partition is stored by +one or several nodes in the cluster +(see [`replication_factor`](@/documentation/reference-manual/configuration.md#replication_factor)). +The layout determines the correspondence between these partitions, +which exist on a logical level, and actual storage nodes. + +## How cluster layouts work in Garage + +A cluster layout is composed of the following components: + +- a table of roles assigned to nodes, defined by the user +- an optimal assignation of partitions to nodes, computed by an algorithm that is ran once when calling `garage layout apply` or the ApplyClusterLayout API endpoint +- a version number + +Garage nodes will always use the cluster layout with the highest version number. + +Garage nodes also maintain and synchronize between them a set of proposed role +changes that haven't yet been applied. These changes will be applied (or +canceled) in the next version of the layout. + +All operations on the layout can be realized using the `garage` CLI or using the +[administration API endpoint](@/documentation/reference-manual/admin-api.md). +We give here a description of CLI commands, the admin API semantics are very similar. + +The following commands insert modifications to the set of proposed role changes +for the next layout version (but they do not create the new layout immediately): + +```bash +garage layout assign [...] +garage layout remove [...] +``` + +The following command can be used to inspect the layout that is currently set in the cluster +and the changes proposed for the next layout version, if any: + +```bash +garage layout show +``` + +The following commands create a new layout with the specified version number, +that either takes into account the proposed changes or cancels them: + +```bash +garage layout apply --version +garage layout revert --version +``` + +The version number of the new layout to create must be 1 + the version number +of the previous layout that existed in the cluster. The `apply` and `revert` +commands will fail otherwise. + +## Warnings about Garage cluster layout management + +**⚠️ Never make several calls to `garage layout apply` or `garage layout +revert` with the same value of the `--version` flag. Doing so can lead to the +creation of several different layouts with the same version number, in which +case your Garage cluster will become inconsistent until fixed.** If a call to +`garage layout apply` or `garage layout revert` has failed and `garage layout +show` indicates that a new layout with the given version number has not been +set in the cluster, then it is fine to call the command again with the same +version number. + +If you are using the `garage` CLI by typing individual commands in your +shell, you shouldn't have much issues as long as you run commands one after +the other and take care of checking the output of `garage layout show` +before applying any changes. + +If you are using the `garage` CLI or the admin API to script layout changes, +follow the following recommendations: + +- If using the CLI, make all of your `garage` CLI calls to the same RPC host. + If using the admin API, make all of your API calls to the same Garage node. Do + not connect to individual nodes to send them each a piece of the layout changes + you are making, as the changes propagate asynchronously between nodes and might + not all be taken into account at the time when the new layout is applied. + +- **Only call `garage layout apply`/ApplyClusterLayout once**, and call it + **strictly after** all of the `layout assign` and `layout remove` + commands/UpdateClusterLayout API calls have returned. + + +## Understanding unexpected layout calculations + +When adding, removing or modifying nodes in a cluster layout, sometimes +unexpected assignations of partitions to node can occur. These assignations +are in fact normal and logical, given the objectives of the algorithm. Indeed, +**the layout algorithm prioritizes moving less data between nodes over +achieving equal distribution of load. It also tries to use all links between +pairs of nodes in equal proportions when moving data.** This section presents +two examples and illustrates how one can control Garage's behavior to obtain +the desired results. + +### Example 1 + +In this example, a cluster is originally composed of 3 nodes in 3 different +zones (data centers). The three nodes are of equal capacity, therefore they +are all fully exploited and all store a copy of all of the data in the cluster. + +Then, a fourth node of the same size is added in the datacenter `dc1`. +As illustrated by the following, **Garage will by default not store any data on the new node**: + +``` +$ garage layout show +==== CURRENT CLUSTER LAYOUT ==== +ID Tags Zone Capacity Usable capacity +b10c110e4e854e5a node1 dc1 1000.0 MB 1000.0 MB (100.0%) +a235ac7695e0c54d node2 dc2 1000.0 MB 1000.0 MB (100.0%) +62b218d848e86a64 node3 dc3 1000.0 MB 1000.0 MB (100.0%) + +Zone redundancy: maximum + +Current cluster layout version: 6 + +==== STAGED ROLE CHANGES ==== +ID Tags Zone Capacity +a11c7cf18af29737 node4 dc1 1000.0 MB + + +==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ==== +ID Tags Zone Capacity Usable capacity +b10c110e4e854e5a node1 dc1 1000.0 MB 1000.0 MB (100.0%) +a11c7cf18af29737 node4 dc1 1000.0 MB 0 B (0.0%) +a235ac7695e0c54d node2 dc2 1000.0 MB 1000.0 MB (100.0%) +62b218d848e86a64 node3 dc3 1000.0 MB 1000.0 MB (100.0%) + +Zone redundancy: maximum + +==== COMPUTATION OF A NEW PARTITION ASSIGNATION ==== + +Partitions are replicated 3 times on at least 3 distinct zones. + +Optimal partition size: 3.9 MB (3.9 MB in previous layout) +Usable capacity / total cluster capacity: 3.0 GB / 4.0 GB (75.0 %) +Effective capacity (replication factor 3): 1000.0 MB + +A total of 0 new copies of partitions need to be transferred. + +dc1 Tags Partitions Capacity Usable capacity + b10c110e4e854e5a node1 256 (0 new) 1000.0 MB 1000.0 MB (100.0%) + a11c7cf18af29737 node4 0 (0 new) 1000.0 MB 0 B (0.0%) + TOTAL 256 (256 unique) 2.0 GB 1000.0 MB (50.0%) + +dc2 Tags Partitions Capacity Usable capacity + a235ac7695e0c54d node2 256 (0 new) 1000.0 MB 1000.0 MB (100.0%) + TOTAL 256 (256 unique) 1000.0 MB 1000.0 MB (100.0%) + +dc3 Tags Partitions Capacity Usable capacity + 62b218d848e86a64 node3 256 (0 new) 1000.0 MB 1000.0 MB (100.0%) + TOTAL 256 (256 unique) 1000.0 MB 1000.0 MB (100.0%) +``` + +While unexpected, this is logical because of the following facts: + +- storing some data on the new node does not help increase the total quantity + of data that can be stored on the cluster, as the two other zones (`dc2` and + `dc3`) still need to store a full copy of everything, and their capacity is + still the same; + +- there is therefore no need to move any data on the new node as this would be pointless; + +- moving data to the new node has a cost which the algorithm decides to not pay if not necessary. + +This distribution of data can however not be what the administrator wanted: if +they added a new node to `dc1`, it might be because the existing node is too +slow, and they wish to divide its load by half. In that case, what they need to +do to force Garage to distribute the data between the two nodes is to attribute +only half of the capacity to each node in `dc1` (in our example, 500M instead of 1G). +In that case, Garage would determine that to be able to store 1G in total, it +would need to store 500M on the old node and 500M on the added one. + + +### Example 2 + +The following example is a slightly different scenario, where `dc1` had two +nodes that were used at 50%, and `dc2` and `dc3` each have one node that is +100% used. All node capacities are the same. + +Then, a node from `dc1` is moved into `dc3`. One could expect that the roles of +`dc1` and `dc3` would simply be swapped: the remaining node in `dc1` would be +used at 100%, and the two nodes now in `dc3` would be used at 50%. Instead, +this happens: + +``` +==== CURRENT CLUSTER LAYOUT ==== +ID Tags Zone Capacity Usable capacity +b10c110e4e854e5a node1 dc1 1000.0 MB 500.0 MB (50.0%) +a11c7cf18af29737 node4 dc1 1000.0 MB 500.0 MB (50.0%) +a235ac7695e0c54d node2 dc2 1000.0 MB 1000.0 MB (100.0%) +62b218d848e86a64 node3 dc3 1000.0 MB 1000.0 MB (100.0%) + +Zone redundancy: maximum + +Current cluster layout version: 8 + +==== STAGED ROLE CHANGES ==== +ID Tags Zone Capacity +a11c7cf18af29737 node4 dc3 1000.0 MB + + +==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ==== +ID Tags Zone Capacity Usable capacity +b10c110e4e854e5a node1 dc1 1000.0 MB 1000.0 MB (100.0%) +a235ac7695e0c54d node2 dc2 1000.0 MB 1000.0 MB (100.0%) +62b218d848e86a64 node3 dc3 1000.0 MB 753.9 MB (75.4%) +a11c7cf18af29737 node4 dc3 1000.0 MB 246.1 MB (24.6%) + +Zone redundancy: maximum + +==== COMPUTATION OF A NEW PARTITION ASSIGNATION ==== + +Partitions are replicated 3 times on at least 3 distinct zones. + +Optimal partition size: 3.9 MB (3.9 MB in previous layout) +Usable capacity / total cluster capacity: 3.0 GB / 4.0 GB (75.0 %) +Effective capacity (replication factor 3): 1000.0 MB + +A total of 128 new copies of partitions need to be transferred. + +dc1 Tags Partitions Capacity Usable capacity + b10c110e4e854e5a node1 256 (128 new) 1000.0 MB 1000.0 MB (100.0%) + TOTAL 256 (256 unique) 1000.0 MB 1000.0 MB (100.0%) + +dc2 Tags Partitions Capacity Usable capacity + a235ac7695e0c54d node2 256 (0 new) 1000.0 MB 1000.0 MB (100.0%) + TOTAL 256 (256 unique) 1000.0 MB 1000.0 MB (100.0%) + +dc3 Tags Partitions Capacity Usable capacity + 62b218d848e86a64 node3 193 (0 new) 1000.0 MB 753.9 MB (75.4%) + a11c7cf18af29737 node4 63 (0 new) 1000.0 MB 246.1 MB (24.6%) + TOTAL 256 (256 unique) 2.0 GB 1000.0 MB (50.0%) +``` + +As we can see, the node that was moved to `dc3` (node4) is only used at 25% (approximatively), +whereas the node that was already in `dc3` (node3) is used at 75%. + +This can be explained by the following: + +- node1 will now be the only node remaining in `dc1`, thus it has to store all + of the data in the cluster. Since it was storing only half of it before, it has + to retrieve the other half from other nodes in the cluster. + +- The data which it does not have is entirely stored by the other node that was + in `dc1` and that is now in `dc3` (node4). There is also a copy of it on node2 + and node3 since both these nodes have a copy of everything. + +- node3 and node4 are the two nodes that will now be in a datacenter that is + under-utilized (`dc3`), this means that those are the two candidates from which + data can be removed to be moved to node1. + +- Garage will move data in equal proportions from all possible sources, in this + case it means that it will tranfer 25% of the entire data set from node3 to + node1 and another 25% from node4 to node1. + +This explains why node3 ends with 75% utilization (100% from before minus 25% +that is moved to node1), and node4 ends with 25% (50% from before minus 25% +that is moved to node1). + +This illustrates the second principle of the layout computation: **if there is +a choice in moving data out of some nodes, then all links between pairs of +nodes are used in equal proportions** (this is approximately true, there is +randomness in the algorithm to achieve this so there might be some small +fluctuations, as we see above). diff --git a/doc/book/operations/multi-hdd.md b/doc/book/operations/multi-hdd.md new file mode 100644 index 00000000..1cbcd805 --- /dev/null +++ b/doc/book/operations/multi-hdd.md @@ -0,0 +1,101 @@ ++++ +title = "Multi-HDD support" +weight = 15 ++++ + + +Since v0.9, Garage natively supports nodes that have several storage drives +for storing data blocks (not for metadata storage). + +## Initial setup + +To set up a new Garage storage node with multiple HDDs, +format and mount all your drives in different directories, +and use a Garage configuration as follows: + +```toml +data_dir = [ + { path = "/path/to/hdd1", capacity = "2T" }, + { path = "/path/to/hdd2", capacity = "4T" }, +] +``` + +Garage will automatically balance all blocks stored by the node +among the different specified directories, proportionally to the +specified capacities. + +## Updating the list of storage locations + +If you add new storage locations to your `data_dir`, +Garage will not rebalance existing data between storage locations. +Newly written blocks will be balanced proportionally to the specified capacities, +and existing data may be moved between drives to improve balancing, +but only opportunistically when a data block is re-written (e.g. an object +is re-uploaded, or an object with a duplicate block is uploaded). + +To understand precisely what is happening, we need to dive in to how Garage +splits data among the different storage locations. + +First of all, Garage divides the set of all possible block hashes +in a fixed number of slices (currently 1024), and assigns +to each slice a primary storage location among the specified data directories. +The number of slices having their primary location in each data directory +is proportionnal to the capacity specified in the config file. + +When Garage receives a block to write, it will always write it in the primary +directory of the slice that contains its hash. + +Now, to be able to not lose existing data blocks when storage locations +are added, Garage also keeps a list of secondary data directories +for all of the hash slices. Secondary data directories for a slice indicates +storage locations that once were primary directories for that slice, i.e. where +Garage knows that data blocks of that slice might be stored. +When Garage is requested to read a certain data block, +it will first look in the primary storage directory of its slice, +and if it doesn't find it there it goes through all of the secondary storage +locations until it finds it. This allows Garage to continue operating +normally when storage locations are added, without having to shuffle +files between drives to place them in the correct location. + +This relatively simple strategy works well but does not ensure that data +is correctly balanced among drives according to their capacity. +To rebalance data, two strategies can be used: + +- Lazy rebalancing: when a block is re-written (e.g. the object is re-uploaded), + Garage checks whether the existing copy is in the primary directory of the slice + or in a secondary directory. If the current copy is in a secondary directory, + Garage re-writes a copy in the primary directory and deletes the one from the + secondary directory. This might never end up rebalancing everything if there + are data blocks that are only read and never written. + +- Active rebalancing: an operator of a Garage node can explicitly launch a repair + procedure that rebalances the data directories, moving all blocks to their + primary location. Once done, all secondary locations for all hash slices are + removed so that they won't be checked anymore when looking for a data block. + +## Read-only storage locations + +If you would like to move all data blocks from an existing data directory to one +or several new data directories, mark the old directory as read-only: + +```toml +data_dir = [ + { path = "/path/to/old_data", read_only = true }, + { path = "/path/to/new_hdd1", capacity = "2T" }, + { path = "/path/to/new_hdd2", capacity = "4T" }, +] +``` + +Garage will be able to read requested blocks from the read-only directory. +Garage will also move data out of the read-only directory either progressively +(lazy rebalancing) or if requested explicitly (active rebalancing). + +Once an active rebalancing has finished, your read-only directory should be empty: +it might still contain subdirectories, but no data files. You can check that +it contains no files using: + +```bash +find -type f /path/to/old_data # should not print anything +``` + +at which point it can be removed from the `data_dir` list in your config file. diff --git a/doc/book/cookbook/recovering.md b/doc/book/operations/recovering.md similarity index 65% rename from doc/book/cookbook/recovering.md rename to doc/book/operations/recovering.md index 2129a7f3..05322b67 100644 --- a/doc/book/cookbook/recovering.md +++ b/doc/book/operations/recovering.md @@ -1,11 +1,11 @@ +++ title = "Recovering from failures" -weight = 50 +weight = 40 +++ Garage is meant to work on old, second-hand hardware. In particular, this makes it likely that some of your drives will fail, and some manual intervention will be needed. -Fear not! For Garage is fully equipped to handle drive failures, in most common cases. +Fear not! Garage is fully equipped to handle drive failures, in most common cases. ## A note on availability of Garage @@ -61,7 +61,7 @@ garage repair -a --yes blocks This will re-synchronize blocks of data that are missing to the new HDD, reading them from copies located on other nodes. -You can check on the advancement of this process by doing the following command: +You can check on the advancement of this process by doing the following command: ```bash garage stats -a @@ -108,3 +108,57 @@ garage layout apply # once satisfied, apply the changes Garage will then start synchronizing all required data on the new node. This process can be monitored using the `garage stats -a` command. + +## Replacement scenario 3: corrupted metadata {#corrupted_meta} + +In some cases, your metadata DB file might become corrupted, for instance if +your node suffered a power outage and did not shut down properly. In this case, +you can recover without having to change the node ID and rebuilding a cluster +layout. This means that data blocks will not need to be shuffled around, you +must simply find a way to repair the metadata file. The best way is generally +to discard the corrupted file and recover it from another source. + +First of all, start by locating the database file in your metadata directory, +which [depends on your `db_engine` +choice](@/documentation/reference-manual/configuration.md#db_engine). Then, +your recovery options are as follows: + +- **Option 1: resyncing from other nodes.** In case your cluster is replicated + with two or three copies, you can simply delete the database file, and Garage + will resync from other nodes. To do so, stop Garage, delete the database file + or directory, and restart Garage. Then, do a full table repair by calling + `garage repair -a --yes tables`. This will take a bit of time to complete as + the new node will need to receive copies of the metadata tables from the + network. + +- **Option 2: restoring a snapshot taken by Garage.** Since v0.9.4, Garage can + [automatically take regular + snapshots](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval) + of your metadata DB file. This file or directory should be located under + `/snapshots`, and is named according to the UTC time at which it + was taken. Stop Garage, discard the database file/directory and replace it by the + snapshot you want to use. For instance, in the case of LMDB: + + ```bash + cd $METADATA_DIR + mv db.lmdb db.lmdb.bak + cp -r snapshots/2024-03-15T12:13:52Z db.lmdb + ``` + + And for Sqlite: + + ```bash + cd $METADATA_DIR + mv db.sqlite db.sqlite.bak + cp snapshots/2024-03-15T12:13:52Z db.sqlite + ``` + + Then, restart Garage and run a full table repair by calling `garage repair -a + --yes tables`. This should run relatively fast as only the changes that + occurred since the snapshot was taken will need to be resynchronized. Of + course, if your cluster is not replicated, you will lose all changes that + occurred since the snapshot was taken. + +- **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or + BTRFS to snapshot your metadata partition, refer to their specific + documentation on rolling back or copying files from an old snapshot. diff --git a/doc/book/cookbook/upgrading.md b/doc/book/operations/upgrading.md similarity index 80% rename from doc/book/cookbook/upgrading.md rename to doc/book/operations/upgrading.md index 9d60a988..a3d2bcf5 100644 --- a/doc/book/cookbook/upgrading.md +++ b/doc/book/operations/upgrading.md @@ -1,6 +1,6 @@ +++ title = "Upgrading Garage" -weight = 60 +weight = 10 +++ Garage is a stateful clustered application, where all nodes are communicating together and share data structures. @@ -9,7 +9,7 @@ On a new version release, there is 2 possibilities: - protocols and data structures remained the same ➡️ this is a **minor upgrade** - protocols or data structures changed ➡️ this is a **major upgrade** -You can quickly now what type of update you will have to operate by looking at the version identifier: +You can quickly know what type of update you will have to operate by looking at the version identifier: when we require our users to do a major upgrade, we will always bump the first nonzero component of the version identifier (e.g. from v0.7.2 to v0.8.0). Conversely, for versions that only require a minor upgrade, the first nonzero component will always stay the same (e.g. from v0.8.0 to v0.8.1). @@ -58,7 +58,7 @@ From a high level perspective, a major upgrade looks like this: ### Major upgarades with minimal downtime -There is only one operation that has to be coordinated cluster-wide: the passage of one version of the internal RPC protocol to the next. +There is only one operation that has to be coordinated cluster-wide: the switch of one version of the internal RPC protocol to the next. This means that an upgrade with very limited downtime can simply be performed from one major version to the next by restarting all nodes simultaneously in the new version. The downtime will simply be the time required for all nodes to stop and start again, which should be less than a minute. @@ -71,7 +71,19 @@ The entire procedure would look something like this: 2. Take each node offline individually to back up its metadata folder, bring them back online once the backup is done. You can do all of the nodes in a single zone at once as that won't impact global cluster availability. - Do not try to make a backup of the metadata folder of a running node. + Do not try to manually copy the metadata folder of a running node. + + **Since Garage v0.9.4,** you can use the `garage meta snapshot --all` command + to take a simultaneous snapshot of the metadata database files of all your + nodes. This avoids the tedious process of having to take them down one by + one before upgrading. Be careful that if automatic snapshotting is enabled, + Garage only keeps the last two snapshots and deletes older ones, so you might + want to disable automatic snapshotting in your upgraded configuration file + until you have confirmed that the upgrade ran successfully. In addition to + snapshotting the metadata databases of your nodes, you should back-up at + least the `cluster_layout` file of one of your Garage instances (this file + should be the same on all nodes and you can copy it safely while Garage is + running). 3. Prepare your binaries and configuration files for the new Garage version @@ -80,6 +92,6 @@ The entire procedure would look something like this: 5. If any specific migration procedure is required, it is usually in one of the two cases: - It can be run on online nodes after the new version has started, during regular cluster operation. - - it has to be run offline + - it has to be run offline, in which case you will have to again take all nodes offline one after the other to run the repair For this last step, please refer to the specific documentation pertaining to the version upgrade you are doing. diff --git a/doc/book/quick-start/_index.md b/doc/book/quick-start/_index.md index ac55d2f7..633b785a 100644 --- a/doc/book/quick-start/_index.md +++ b/doc/book/quick-start/_index.md @@ -1,6 +1,6 @@ +++ title = "Quick Start" -weight = 0 +weight = 10 sort_by = "weight" template = "documentation.html" +++ @@ -35,10 +35,20 @@ Place this binary somewhere in your `$PATH` so that you can invoke the `garage` command directly (for instance you can copy the binary in `/usr/local/bin` or in `~/.local/bin`). +You may also check whether your distribution already includes a +[binary package for Garage](@/documentation/cookbook/binary-packages.md). + If a binary of the last version is not available for your architecture, or if you want a build customized for your system, you can [build Garage from source](@/documentation/cookbook/from-source.md). +If none of these option work for you, you can also run Garage in a Docker +container. When using Docker, the commands used in this guide will not work +anymore. We recommend reading the tutorial on [configuring a +multi-node cluster](@/documentation/cookbook/real-world.md) to learn about +using Garage as a Docker container. For simplicity, a minimal command to launch +Garage using Docker is provided in this quick start guide as well. + ## Configuring and starting Garage @@ -54,9 +64,9 @@ to generate unique and private secrets for security reasons: cat > garage.toml < +garage layout assign -z dc1 -c 1G ``` where `` corresponds to the identifier of the node shown by `garage status` (first column). You can enter simply a prefix of that identifier. -For instance here you could write just `garage layout assign -z dc1 -c 1 563e`. +For instance here you could write just `garage layout assign -z dc1 -c 1G 563e`. The layout then has to be applied to the cluster, using: ```bash -garage layout apply +garage layout apply --version 1 ``` @@ -206,7 +246,7 @@ one key can access multiple buckets, multiple keys can access one bucket. Create an API key using the following command: ``` -garage key new --name nextcloud-app-key +garage key create nextcloud-app-key ``` The output should look as follows: @@ -245,7 +285,7 @@ garage bucket info nextcloud-bucket ``` -## Uploading and downlading from Garage +## Uploading and downloading from Garage To download and upload files on garage, we can use a third-party tool named `awscli`. @@ -266,12 +306,14 @@ named `~/.awsrc` with this content: export AWS_ACCESS_KEY_ID=xxxx # put your Key ID here export AWS_SECRET_ACCESS_KEY=xxxx # put your Secret key here export AWS_DEFAULT_REGION='garage' -export AWS_ENDPOINT='http://localhost:3900' +export AWS_ENDPOINT_URL='http://localhost:3900' -function aws { command aws --endpoint-url $AWS_ENDPOINT $@ ; } aws --version ``` +Note you need to have at least `awscli` `>=1.29.0` or `>=2.13.0`, otherwise you +need to specify `--endpoint-url` explicitly on each `awscli` invocation. + Now, each time you want to use `awscli` on this target, run: ```bash @@ -290,13 +332,13 @@ sourcing the right file.* aws s3 ls # list objects of a bucket -aws s3 ls s3://my_files +aws s3 ls s3://nextcloud-bucket # copy from your filesystem to garage -aws s3 cp /proc/cpuinfo s3://my_files/cpuinfo.txt +aws s3 cp /proc/cpuinfo s3://nextcloud-bucket/cpuinfo.txt # copy from garage to your filesystem -aws s3 cp s3/my_files/cpuinfo.txt /tmp/cpuinfo.txt +aws s3 cp s3://nextcloud-bucket/cpuinfo.txt /tmp/cpuinfo.txt ``` Note that you can use `awscli` for more advanced operations like @@ -308,7 +350,7 @@ Check [our s3 compatibility list](@/documentation/reference-manual/s3-compatibil ### Other tools for interacting with Garage -The following tools can also be used to send and recieve files from/to Garage: +The following tools can also be used to send and receive files from/to Garage: - [minio-client](@/documentation/connect/cli.md#minio-client) - [s3cmd](@/documentation/connect/cli.md#s3cmd) diff --git a/doc/book/reference-manual/_index.md b/doc/book/reference-manual/_index.md index ab1de5e6..1f360e57 100644 --- a/doc/book/reference-manual/_index.md +++ b/doc/book/reference-manual/_index.md @@ -1,6 +1,6 @@ +++ title = "Reference Manual" -weight = 5 +weight = 60 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/reference-manual/admin-api.md b/doc/book/reference-manual/admin-api.md index 363bc886..fcf49e8c 100644 --- a/doc/book/reference-manual/admin-api.md +++ b/doc/book/reference-manual/admin-api.md @@ -8,18 +8,21 @@ listen address is specified in the `[admin]` section of the configuration file (see [configuration file reference](@/documentation/reference-manual/configuration.md)) -**WARNING.** At this point, there is no comittement to stability of the APIs described in this document. -We will bump the version numbers prefixed to each API endpoint at each time the syntax +**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document. +We will bump the version numbers prefixed to each API endpoint each time the syntax or semantics change, meaning that code that relies on these endpoint will break when changes are introduced. -The Garage administration API was introduced in version 0.7.2, this document -does not apply to older versions of Garage. +Versions: + - Before Garage 0.7.2 - no admin API + - Garage 0.7.2 - admin APIv0 + - Garage 0.9.0 - admin APIv1, deprecate admin APIv0 + ## Access control -The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: +The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without @@ -39,15 +42,101 @@ Authorization: Bearer ## Administration API endpoints -### Metrics-related endpoints - -#### Metrics `GET /metrics` +### Metrics `GET /metrics` Returns internal Garage metrics in Prometheus format. +The metrics are directly documented when returned by the API. + +**Example:** + +``` +$ curl -i http://localhost:3903/metrics +HTTP/1.1 200 OK +content-type: text/plain; version=0.0.4 +content-length: 12145 +date: Tue, 08 Aug 2023 07:25:05 GMT + +# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors +# TYPE api_admin_error_counter counter +api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1 +api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3 +# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints +# TYPE api_admin_request_counter counter +api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7 +api_admin_request_counter{api_endpoint="Health"} 3 +# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints +... +``` + +### Health `GET /health` + +Returns `200 OK` if enough nodes are up to have a quorum (ie. serve requests), +otherwise returns `503 Service Unavailable`. + +**Example:** + +``` +$ curl -i http://localhost:3903/health +HTTP/1.1 200 OK +content-type: text/plain +content-length: 102 +date: Tue, 08 Aug 2023 07:22:38 GMT + +Garage is fully operational +Consult the full health check API endpoint at /v0/health for more details +``` + +### On-demand TLS `GET /check` + +To prevent abuse for on-demand TLS, Caddy developers have specified an endpoint that can be queried by the reverse proxy +to know if a given domain is allowed to get a certificate. Garage implements these endpoints to tell if a given domain is handled by Garage or is garbage. + +Garage responds with the following logic: + - If the domain matches the pattern `.`, returns 200 OK + - If the domain matches the pattern `.` and website is configured for ``, returns 200 OK + - If the domain matches the pattern `` and website is configured for ``, returns 200 OK + - Otherwise, returns 404 Not Found, 400 Bad Request or 5xx requests. + +*Note 1: because in the path-style URL mode, there is only one domain that is not known by Garage, hence it is not supported by this API endpoint. +You must manually declare the domain in your reverse-proxy. Idem for K2V.* + +*Note 2: buckets in a user's namespace are not supported yet by this endpoint. This is a limitation of this endpoint currently.* + +**Example:** Suppose a Garage instance is configured with `s3_api.root_domain = .s3.garage.localhost` and `s3_web.root_domain = .web.garage.localhost`. + +With a private `media` bucket (name in the global namespace, website is disabled), the endpoint will feature the following behavior: + +``` +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media.s3.garage.localhost +200 +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media +400 +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media.web.garage.localhost +400 +``` + +With a public `example.com` bucket (name in the global namespace, website is activated), the endpoint will feature the following behavior: + +``` +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com.s3.garage.localhost +200 +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com +200 +$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com.web.garage.localhost +200 +``` + + +**References:** + - [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls) + - [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939) + - [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179) ### Cluster operations -These endpoints are defined on a dedicated [Redocly page](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html). You can also download its [OpenAPI specification](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml). +These endpoints have a dedicated OpenAPI spec. + - APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml) + - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml) Requesting the API from the command line can be as simple as running: diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 38062bab..1f583fe6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -8,39 +8,60 @@ weight = 20 Here is an example `garage.toml` configuration file that illustrates all of the possible options: ```toml +replication_factor = 3 +consistency_mode = "consistent" + metadata_dir = "/var/lib/garage/meta" data_dir = "/var/lib/garage/data" +metadata_snapshots_dir = "/var/lib/garage/snapshots" +metadata_fsync = true +data_fsync = false +disable_scrub = false +use_local_tz = false +metadata_auto_snapshot_interval = "6h" db_engine = "lmdb" -block_size = 1048576 - -sled_cache_capacity = 134217728 -sled_flush_every_ms = 2000 - -replication_mode = "3" +block_size = "1M" +block_ram_buffer_max = "256MiB" +block_max_concurrent_reads = 16 +block_max_concurrent_writes_per_request =10 +lmdb_map_size = "1T" compression_level = 1 rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6" rpc_bind_addr = "[::]:3901" +rpc_bind_outgoing = false rpc_public_addr = "[fc00:1::1]:3901" +# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet: +# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64" + + +allow_world_readable_secrets = false bootstrap_peers = [ "563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901", - "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332[fc00:1::2]:3901", + "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901", "681456ab91350f92242e80a531a3ec9392cb7c974f72640112f90a600d7921a4@[fc00:B::1]:3901", "212fd62eeaca72c122b45a7f4fa0f55e012aa5e24ac384a72a3016413fa724ff@[fc00:F::1]:3901", ] +allow_punycode = false [consul_discovery] +api = "catalog" consul_http_addr = "http://127.0.0.1:8500" service_name = "garage-daemon" ca_cert = "/etc/consul/consul-ca.crt" client_cert = "/etc/consul/consul-client.crt" client_key = "/etc/consul/consul-key.crt" +# for `agent` API mode, unset client_cert and client_key, and optionally enable `token` +# token = "abcdef-01234-56789" tls_skip_verify = false +tags = [ "dns-enabled" ] +meta = { dns-acl = "allow trusted" } + [kubernetes_discovery] namespace = "garage" @@ -56,11 +77,12 @@ root_domain = ".s3.garage" [s3_web] bind_addr = "[::]:3902" root_domain = ".web.garage" +add_host_to_metrics = true [admin] api_bind_addr = "0.0.0.0:3903" -metrics_token = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81" -admin_token = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a" +metrics_token = "BCAdFjoa9G0KJR0WXnHHm7fs1ZAbfpI8iIZ+Z/a2NgI=" +admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=" trace_sink = "http://localhost:4317" ``` @@ -68,97 +90,100 @@ The following gives details about each available configuration option. ## Available configuration options -### `metadata_dir` +### Index -The directory in which Garage will store its metadata. This contains the node identifier, -the network configuration and the peer list, the list of buckets and keys as well -as the index of all objects, object version and object blocks. +[Environment variables](#env_variables). -Store this folder on a fast SSD drive if possible to maximize Garage's performance. +Top-level configuration options, in alphabetical order: +[`allow_punycode`](#allow_punycode), +[`allow_world_readable_secrets`](#allow_world_readable_secrets), +[`block_max_concurrent_reads`](`block_max_concurrent_reads), +[`block_ram_buffer_max`](#block_ram_buffer_max), +[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request), +[`block_size`](#block_size), +[`bootstrap_peers`](#bootstrap_peers), +[`compression_level`](#compression_level), +[`consistency_mode`](#consistency_mode), +[`data_dir`](#data_dir), +[`data_fsync`](#data_fsync), +[`db_engine`](#db_engine), +[`disable_scrub`](#disable_scrub), +[`lmdb_map_size`](#lmdb_map_size), +[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval), +[`metadata_dir`](#metadata_dir), +[`metadata_fsync`](#metadata_fsync), +[`metadata_snapshots_dir`](#metadata_snapshots_dir), +[`replication_factor`](#replication_factor), +[`rpc_bind_addr`](#rpc_bind_addr), +[`rpc_bind_outgoing`](#rpc_bind_outgoing), +[`rpc_public_addr`](#rpc_public_addr), +[`rpc_public_addr_subnet`](#rpc_public_addr_subnet) +[`rpc_secret`/`rpc_secret_file`](#rpc_secret), +[`use_local_tz`](#use_local_tz). -### `data_dir` +The `[consul_discovery]` section: +[`api`](#consul_api), +[`ca_cert`](#consul_ca_cert), +[`client_cert`](#consul_client_cert_and_key), +[`client_key`](#consul_client_cert_and_key), +[`consul_http_addr`](#consul_http_addr), +[`meta`](#consul_tags_and_meta), +[`service_name`](#consul_service_name), +[`tags`](#consul_tags_and_meta), +[`tls_skip_verify`](#consul_tls_skip_verify), +[`token`](#consul_token). -The directory in which Garage will store the data blocks of objects. -This folder can be placed on an HDD. The space available for `data_dir` -should be counted to determine a node's capacity -when [adding it to the cluster layout](@/documentation/cookbook/real-world.md). +The `[kubernetes_discovery]` section: +[`namespace`](#kube_namespace), +[`service_name`](#kube_service_name), +[`skip_crd`](#kube_skip_crd). -### `db_engine` (since `v0.8.0`) +The `[s3_api]` section: +[`api_bind_addr`](#s3_api_bind_addr), +[`root_domain`](#s3_root_domain), +[`s3_region`](#s3_region). -By default, Garage uses the Sled embedded database library -to store its metadata on-disk. Since `v0.8.0`, Garage can use alternative storage backends as follows: +The `[s3_web]` section: +[`add_host_to_metrics`](#web_add_host_to_metrics), +[`bind_addr`](#web_bind_addr), +[`root_domain`](#web_root_domain). -| DB engine | `db_engine` value | Database path | -| --------- | ----------------- | ------------- | -| [Sled](https://sled.rs) | `"sled"` | `/db/` | -| [LMDB](https://www.lmdb.tech) | `"lmdb"` | `/db.lmdb/` | -| [Sqlite](https://sqlite.org) | `"sqlite"` | `/db.sqlite` | +The `[admin]` section: +[`api_bind_addr`](#admin_api_bind_addr), +[`metrics_token`/`metrics_token_file`](#admin_metrics_token), +[`admin_token`/`admin_token_file`](#admin_token), +[`trace_sink`](#admin_trace_sink), -Performance characteristics of the different DB engines are as follows: +### Environment variables {#env_variables} -- Sled: the default database engine, which tends to produce - large data files and also has performance issues, especially when the metadata folder - is on a traditional HDD and not on SSD. -- LMDB: the recommended alternative on 64-bit systems, - much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable - between architectures, so for instance the Garage database of an x86-64 - node cannot be moved to an ARM64 node. Also note that, while LMDB can technically be used on 32-bit systems, - this will limit your node to very small database sizes due to how LMDB works; it is therefore not recommended. -- Sqlite: Garage supports Sqlite as a storage backend for metadata, - however it may have issues and is also very slow in its current implementation, - so it is not recommended to be used for now. +The following configuration parameters must be specified as environment variables, +they do not exist in the configuration file: -It is possible to convert Garage's metadata directory from one format to another with a small utility named `convert_db`, -which can be downloaded at the following locations: -[for amd64](https://garagehq.deuxfleurs.fr/_releases/convert_db/amd64/convert_db), -[for i386](https://garagehq.deuxfleurs.fr/_releases/convert_db/i386/convert_db), -[for arm64](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm64/convert_db), -[for arm](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm/convert_db). -The `convert_db` utility is used as folows: +- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the + Garage daemon send its logs to `syslog` (using the libc `syslog` function) + instead of printing to stderr. -``` -convert-db -a -i \ - -b -o -``` +- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the + Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`) + instead of printing to stderr. -Make sure to specify the full database path as presented in the table above, -and not just the path to the metadata directory. +The following environment variables can be used to override the corresponding +values in the configuration file: -### `block_size` +- [`GARAGE_ALLOW_WORLD_READABLE_SECRETS`](#allow_world_readable_secrets) +- [`GARAGE_RPC_SECRET` and `GARAGE_RPC_SECRET_FILE`](#rpc_secret) +- [`GARAGE_ADMIN_TOKEN` and `GARAGE_ADMIN_TOKEN_FILE`](#admin_token) +- [`GARAGE_METRICS_TOKEN` and `GARAGE_METRICS_TOKEN`](#admin_metrics_token) -Garage splits stored objects in consecutive chunks of size `block_size` -(except the last one which might be smaller). The default size is 1MB and -should work in most cases. We recommend increasing it to e.g. 10MB if -you are using Garage to store large files and have fast network connections -between all nodes (e.g. 1gbps). -If you are interested in tuning this, feel free to do so (and remember to -report your findings to us!). When this value is changed for a running Garage -installation, only files newly uploaded will be affected. Previously uploaded -files will remain available. This however means that chunks from existing files -will not be deduplicated with chunks from newly uploaded files, meaning you -might use more storage space that is optimally possible. +### Top-level configuration options -### `sled_cache_capacity` +#### `replication_factor` (since `v1.0.0`) {#replication_factor} -This parameter can be used to tune the capacity of the cache used by -[sled](https://sled.rs), the database Garage uses internally to store metadata. -Tune this to fit the RAM you wish to make available to your Garage instance. -This value has a conservative default (128MB) so that Garage doesn't use too much -RAM by default, but feel free to increase this for higher performance. +The replication factor can be any positive integer smaller or equal the node count in your cluster. +The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics. -### `sled_flush_every_ms` - -This parameters can be used to tune the flushing interval of sled. -Increase this if sled is thrashing your SSD, at the risk of losing more data in case -of a power outage (though this should not matter much as data is replicated on other -nodes). The default value, 2000ms, should be appropriate for most use cases. - -### `replication_mode` - -Garage supports the following replication modes: - -- `none` or `1`: data stored on Garage is stored on a single node. There is no +- `1`: data stored on Garage is stored on a single node. There is no redundancy, and data will be unavailable as soon as one node fails or its network is disconnected. Do not use this for anything else than test deployments. @@ -169,17 +194,6 @@ Garage supports the following replication modes: before losing data. Data remains available in read-only mode when one node is down, but write operations will fail. - - `2-dangerous`: a variant of mode `2`, where written objects are written to - the second replica asynchronously. This means that Garage will return `200 - OK` to a PutObject request before the second copy is fully written (or even - before it even starts being written). This means that data can more easily - be lost if the node crashes before a second copy can be completed. This - also means that written objects might not be visible immediately in read - operations. In other words, this mode severely breaks the consistency and - durability guarantees of standard Garage cluster operation. Benefits of - this mode: you can still write to your cluster when one node is - unavailable. - - `3`: data stored on Garage will be stored on three different nodes, if possible each in a different zones. Garage tolerates two node failure, or several node failures but in no more than two zones (in a deployment with at @@ -187,57 +201,370 @@ Garage supports the following replication modes: or node failures are only in a single zone, reading and writing data to Garage can continue normally. - - `3-degraded`: a variant of replication mode `3`, that lowers the read +- `5`, `7`, ...: When setting the replication factor above 3, it is most useful to + choose an uneven value, since for every two copies added, one more node can fail + before losing the ability to write and read to the cluster. + +Note that in modes `2` and `3`, +if at least the same number of zones are available, an arbitrary number of failures in +any given zone is tolerated as copies of data will be spread over several zones. + +**Make sure `replication_factor` is the same in the configuration files of all nodes. +Never run a Garage cluster where that is not the case.** + +It is technically possible to change the replication factor although it's a +dangerous operation that is not officially supported. This requires you to +delete the existing cluster layout and create a new layout from scratch, +meaning that a full rebalancing of your cluster's data will be needed. To do +it, shut down your cluster entirely, delete the `custer_layout` files in the +meta directories of all your nodes, update all your configuration files with +the new `replication_factor` parameter, restart your cluster, and then create a +new layout with all the nodes you want to keep. Rebalancing data will take +some time, and data might temporarily appear unavailable to your users. +It is recommended to shut down public access to the cluster while rebalancing +is in progress. In theory, no data should be lost as rebalancing is a +routine operation for Garage, although we cannot guarantee you that everything + will go right in such an extreme scenario. + +#### `consistency_mode` (since `v1.0.0`) {#consistency_mode} + +The consistency mode setting determines the read and write behaviour of your cluster. + + - `consistent`: The default setting. This is what the paragraph above describes. + The read and write quorum will be determined so that read-after-write consistency + is guaranteed. + - `degraded`: Lowers the read quorum to `1`, to allow you to read data from your cluster when several nodes (or nodes in several zones) are unavailable. In this mode, Garage - does not provide read-after-write consistency anymore. The write quorum is - still 2, ensuring that data successfully written to Garage is stored on at - least two nodes. - - - `3-dangerous`: a variant of replication mode `3` that lowers both the read + does not provide read-after-write consistency anymore. + The write quorum stays the same as in the `consistent` mode, ensuring that + data successfully written to Garage is stored on multiple nodes (depending + the replication factor). + - `dangerous`: This mode lowers both the read and write quorums to `1`, to allow you to both read and write to your cluster when several nodes (or nodes in several zones) are unavailable. It is the least consistent mode of operation proposed by Garage, and also one that should probably never be used. -Note that in modes `2` and `3`, -if at least the same number of zones are available, an arbitrary number of failures in -any given zone is tolerated as copies of data will be spread over several zones. +Changing the `consistency_mode` between modes while leaving the `replication_factor` untouched +(e.g. setting your node's `consistency_mode` to `degraded` when it was previously unset, or from +`dangerous` to `consistent`), can be done easily by just changing the `consistency_mode` +parameter in your config files and restarting all your Garage nodes. -**Make sure `replication_mode` is the same in the configuration files of all nodes. -Never run a Garage cluster where that is not the case.** +The consistency mode can be used together with various replication factors, to achieve +a wide range of read and write characteristics. Some examples: + + - Replication factor `2`, consistency mode `degraded`: While this mode + technically exists, its properties are the same as with consistency mode `consistent`, + since the read quorum with replication factor `2`, consistency mode `consistent` is already 1. + + - Replication factor `2`, consistency mode `dangerous`: written objects are written to + the second replica asynchronously. This means that Garage will return `200 + OK` to a PutObject request before the second copy is fully written (or even + before it even starts being written). This means that data can more easily + be lost if the node crashes before a second copy can be completed. This + also means that written objects might not be visible immediately in read + operations. In other words, this configuration severely breaks the consistency and + durability guarantees of standard Garage cluster operation. Benefits of + this configuration: you can still write to your cluster when one node is + unavailable. The quorums associated with each replication mode are described below: -| `replication_mode` | Number of replicas | Write quorum | Read quorum | Read-after-write consistency? | -| ------------------ | ------------------ | ------------ | ----------- | ----------------------------- | -| `none` or `1` | 1 | 1 | 1 | yes | -| `2` | 2 | 2 | 1 | yes | -| `2-dangerous` | 2 | 1 | 1 | NO | -| `3` | 3 | 2 | 2 | yes | -| `3-degraded` | 3 | 2 | 1 | NO | -| `3-dangerous` | 3 | 1 | 1 | NO | +| `consistency_mode` | `replication_factor` | Write quorum | Read quorum | Read-after-write consistency? | +| ------------------ | -------------------- | ------------ | ----------- | ----------------------------- | +| `consistent` | 1 | 1 | 1 | yes | +| `consistent` | 2 | 2 | 1 | yes | +| `dangerous` | 2 | 1 | 1 | NO | +| `consistent` | 3 | 2 | 2 | yes | +| `degraded` | 3 | 2 | 1 | NO | +| `dangerous` | 3 | 1 | 1 | NO | -Changing the `replication_mode` between modes with the same number of replicas -(e.g. from `3` to `3-degraded`, or from `2-dangerous` to `2`), can be done easily by -just changing the `replication_mode` parameter in your config files and restarting all your -Garage nodes. +#### `metadata_dir` {#metadata_dir} -It is also technically possible to change the replication mode to a mode with a -different numbers of replicas, although it's a dangerous operation that is not -officially supported. This requires you to delete the existing cluster layout -and create a new layout from scratch, meaning that a full rebalancing of your -cluster's data will be needed. To do it, shut down your cluster entirely, -delete the `custer_layout` files in the meta directories of all your nodes, -update all your configuration files with the new `replication_mode` parameter, -restart your cluster, and then create a new layout with all the nodes you want -to keep. Rebalancing data will take some time, and data might temporarily -appear unavailable to your users. It is recommended to shut down public access -to the cluster while rebalancing is in progress. In theory, no data should be -lost as rebalancing is a routine operation for Garage, although we cannot -guarantee you that everything will go right in such an extreme scenario. +The directory in which Garage will store its metadata. This contains the node identifier, +the network configuration and the peer list, the list of buckets and keys as well +as the index of all objects, object version and object blocks. -### `compression_level` +Store this folder on a fast SSD drive if possible to maximize Garage's performance. + + +#### `data_dir` {#data_dir} + +The directory in which Garage will store the data blocks of objects. +This folder can be placed on an HDD. The space available for `data_dir` +should be counted to determine a node's capacity +when [adding it to the cluster layout](@/documentation/cookbook/real-world.md). + +Since `v0.9.0`, Garage supports multiple data directories with the following syntax: + +```toml +data_dir = [ + { path = "/path/to/old_data", read_only = true }, + { path = "/path/to/new_hdd1", capacity = "2T" }, + { path = "/path/to/new_hdd2", capacity = "4T" }, +] +``` + +See [the dedicated documentation page](@/documentation/operations/multi-hdd.md) +on how to operate Garage in such a setup. + +#### `metadata_snapshots_dir` (since `v1.1.0`) {#metadata_snapshots_dir} + +The directory in which Garage will store metadata snapshots when it +performs a snapshot of the metadata database, either when instructed to do +so from a RPC call or regularly through +[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval). + +By default, Garage will store snapshots into a `snapshots/` subdirectory +of [`metadata_dir`](#metadata_dir). This might quickly fill up your +metadata storage space if you use snapshots, because Garage will need up +to 4x the space of the existing metadata database: each snapshot requires +roughly as much space as the original database, and Garage temporarily +needs to store up to three different snapshots before it cleans up the oldest +snapshot to go back to two stored snapshots. + +To prevent filling your disk, you might to change this setting to a +directory with ample available space, e.g. on the same storage space as +[`data_dir`](#data_dir). + +#### `db_engine` (since `v0.8.0`) {#db_engine} + +Since `v0.8.0`, Garage can use alternative storage backends as follows: + +| DB engine | `db_engine` value | Database path | +| --------- | ----------------- | ------------- | +| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `/db.lmdb/` | +| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `/db.sqlite` | +| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `/db.fjall/` | +| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `/db/` | + +Sled was supported until Garage v0.9.x, and was removed in Garage v1.0. +You can still use an older binary of Garage (e.g. v0.9.4) to migrate +old Sled metadata databases to another engine. + +Performance characteristics of the different DB engines are as follows: + +- LMDB: the recommended database engine for high-performance distributed clusters. +LMDB works very well, but is known to have the following limitations: + + - The data format of LMDB is not portable between architectures, so for + instance the Garage database of an x86-64 node cannot be moved to an ARM64 + node. + + - While LMDB can technically be used on 32-bit systems, this will limit your + node to very small database sizes due to how LMDB works; it is therefore + not recommended. + + - Several users have reported corrupted LMDB database files after an unclean + shutdown (e.g. a power outage). This situation can generally be recovered + from if your cluster is geo-replicated (by rebuilding your metadata db from + other nodes), or if you have saved regular snapshots at the filesystem + level. + + - Keys in LMDB are limited to 511 bytes. This limit translates to limits on + object keys in S3 and sort keys in K2V that are limted to 479 bytes. + +- Sqlite: Garage supports Sqlite as an alternative storage backend for + metadata, which does not have the issues listed above for LMDB. + On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal + performance, which was fixed with the addition of `metadata_fsync`. + Sqlite is still probably slower than LMDB due to the way we use it, + so it is not the best choice for high-performance storage clusters, + but it should work fine in many cases. + +- Fjall: a storage engine based on LSM trees, which theoretically allow for + higher write throughput than other storage engines that are based on B-trees. + Using Fjall could potentially improve Garage's performance significantly in + write-heavy workloads. **Support for Fjall is experimental at this point**, + we have added it to Garage for evaluation purposes only. **Do not use it for + production-critical workloads.** + + +It is possible to convert Garage's metadata directory from one format to another +using the `garage convert-db` command, which should be used as follows: + +``` +garage convert-db -a -i \ + -b -o +``` + +Make sure to specify the full database path as presented in the table above +(third colummn), and not just the path to the metadata directory. + +#### `metadata_fsync` {#metadata_fsync} + +Whether to enable synchronous mode for the database engine or not. +This is disabled (`false`) by default. + +This reduces the risk of metadata corruption in case of power failures, +at the cost of a significant drop in write performance, +as Garage will have to pause to sync data to disk much more often +(several times for API calls such as PutObject). + +Using this option reduces the risk of simultaneous metadata corruption on several +cluster nodes, which could lead to data loss. + +If multi-site replication is used, this option is most likely not necessary, as +it is extremely unlikely that two nodes in different locations will have a +power failure at the exact same time. + +(Metadata corruption on a single node is not an issue, the corrupted data file +can always be deleted and reconstructed from the other nodes in the cluster.) + +Here is how this option impacts the different database engines: + +| Database | `metadata_fsync = false` (default) | `metadata_fsync = true` | +|----------|------------------------------------|-------------------------------| +| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` | +| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` | +| Fjall | default options | not supported | + +Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`). + +#### `data_fsync` {#data_fsync} + +Whether to `fsync` data blocks and their containing directory after they are +saved to disk. +This is disabled (`false`) by default. + +This might reduce the risk that a data block is lost in rare +situations such as simultaneous node losing power, +at the cost of a moderate drop in write performance. + +Similarly to `metatada_fsync`, this is likely not necessary +if geographical replication is used. + +#### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval} + +If this value is set, Garage will automatically take a snapshot of the metadata +DB file at a regular interval and save it in the metadata directory. +This parameter can take any duration string that can be parsed by +the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate. + +Snapshots can allow to recover from situations where the metadata DB file is +corrupted, for instance after an unclean shutdown. See [this +page](@/documentation/operations/recovering.md#corrupted_meta) for details. +Garage keeps only the two most recent snapshots of the metadata DB and deletes +older ones automatically. + +Note that taking a metadata snapshot is a relatively intensive operation as the +entire data file is copied. A snapshot being taken might have performance +impacts on the Garage node while it is running. If the cluster is under heavy +write load when a snapshot operation is running, this might also cause the +database file to grow in size significantly as pages cannot be recycled easily. +For this reason, it might be better to use filesystem-level snapshots instead +if possible. + +#### `disable_scrub` {#disable_scrub} + +By default, Garage runs a scrub of the data directory approximately once per +month, with a random delay to avoid all nodes running at the same time. When +it scrubs the data directory, Garage will read all of the data files stored on +disk to check their integrity, and will rebuild any data files that it finds +corrupted, using the remaining valid copies stored on other nodes. +See [this page](@/documentation/operations/durability-repairs.md#scrub) for details. + +Set the `disable_scrub` configuration value to `true` if you don't need Garage +to scrub the data directory, for instance if you are already scrubbing at the +filesystem level. Note that in this case, if you find a corrupted data file, +you should delete it from the data directory and then call `garage repair +blocks` on the node to ensure that it re-obtains a copy from another node on +the network. + +#### `use_local_tz` (since `v1.1.0`) {#use_local_tz} + +By default, Garage runs the lifecycle worker every day at midnight in UTC. Set the +`use_local_tz` configuration value to `true` if you want Garage to run the +lifecycle worker at midnight in your local timezone. If you have multiple nodes, +you should also ensure that each node has the same timezone configuration. + +#### `block_size` {#block_size} + +Garage splits stored objects in consecutive chunks of size `block_size` +(except the last one which might be smaller). The default size is 1MiB and +should work in most cases. We recommend increasing it to e.g. 10MiB if +you are using Garage to store large files and have fast network connections +between all nodes (e.g. 1gbps). + +If you are interested in tuning this, feel free to do so (and remember to +report your findings to us!). When this value is changed for a running Garage +installation, only files newly uploaded will be affected. Previously uploaded +files will remain available. This however means that chunks from existing files +will not be deduplicated with chunks from newly uploaded files, meaning you +might use more storage space that is optimally possible. + +#### `block_ram_buffer_max` (since `v0.9.4`) {#block_ram_buffer_max} + +A limit on the total size of data blocks kept in RAM by S3 API nodes awaiting +to be sent to storage nodes asynchronously. + +Explanation: since Garage wants to tolerate node failures, it uses quorum +writes to send data blocks to storage nodes: try to write the block to three +nodes, and return ok as soon as two writes complete. So even if all three nodes +are online, the third write always completes asynchronously. In general, there +are not many writes to a cluster, and the third asynchronous write can +terminate early enough so as to not cause unbounded RAM growth. However, if +the S3 API node is continuously receiving large quantities of data and the +third node is never able to catch up, many data blocks will be kept buffered in +RAM as they are awaiting transfer to the third node. + +The `block_ram_buffer_max` sets a limit to the size of buffers that can be kept +in RAM in this process. When the limit is reached, backpressure is applied +back to the S3 client. + +Note that this only counts buffers that have arrived to a certain stage of +processing (received from the client + encrypted and/or compressed as +necessary) and are ready to send to the storage nodes. Many other buffers will +not be counted and this is not a hard limit on RAM consumption. In particular, +if many clients send requests simultaneously with large objects, the RAM +consumption will always grow linearly with the number of concurrent requests, +as each request will use a few buffers of size `block_size` for receiving and +intermediate processing before even trying to send the data to the storage +node. + +The default value is 256MiB. + +#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads} + +The maximum number of blocks (individual files in the data directory) open +simultaneously for reading. + +Reducing this number does not limit the number of data blocks that can be +transferred through the network simultaneously. This mechanism was just added +as a backpressure mechanism for HDD read speed: it helps avoid a situation +where too many requests are coming in and Garage is reading too many block +files simultaneously, thus not making timely progress on any of the reads. + +When a request to read a data block comes in through the network, the requests +awaits for one of the `block_max_concurrent_reads` slots to be available +(internally implemented using a Semaphore object). Once it acquired a read +slot, it reads the entire block file to RAM and frees the slot as soon as the +block file is finished reading. Only after the slot is released will the +block's data start being transferred over the network. If the request fails to +acquire a reading slot wihtin 15 seconds, it fails with a timeout error. +Timeout events can be monitored through the `block_read_semaphore_timeouts` +metric in Prometheus: a non-zero number of such events indicates an I/O +bottleneck on HDD read speed. + + +#### `block_max_concurrent_writes_per_request` (since `v2.1.0`) {#block_max_concurrent_writes_per_request} + +This parameter is designed to adapt to the concurrent write performance of +different storage media.Maximum number of parallel block writes per put request +Higher values improve throughput but increase memory usage. + +Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + +#### `lmdb_map_size` {#lmdb_map_size} + +This parameters can be used to set the map size used by LMDB, +which is the size of the virtual memory region used for mapping the database file. +The value of this parameter is the maximum size the metadata database can take. +This value is not bound by the physical RAM size of the machine running Garage. +If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines. + +#### `compression_level` {#compression_level} Zstd compression level to use for storing blocks. @@ -245,7 +572,7 @@ Values between `1` (faster compression) and `19` (smaller file) are standard com levels for zstd. From `20` to `22`, compression levels are referred as "ultra" and must be used with extra care as it will use lot of memory. A value of `0` will let zstd choose a default value (currently `3`). Finally, zstd has also compression designed to be faster -than default compression levels, they range from `-1` (smaller file) to `-99` (faster +than default compression levels, they range from `-1` (smaller file) to `-99` (faster compression). If you do not specify a `compression_level` entry, Garage will set it to `1` for you. With @@ -261,7 +588,7 @@ Compression is done synchronously, setting a value too high will add latency to This value can be different between nodes, compression is done by the node which receive the API call. -### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) +#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret} Garage uses a secret key, called an RPC secret, that is shared between all nodes of the cluster in order to identify these nodes and allow them to @@ -273,7 +600,10 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is given in the configuration variable `rpc_secret_file`, or specified as an environment variable `GARAGE_RPC_SECRET`. -### `rpc_bind_addr` +Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file +storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable. + +#### `rpc_bind_addr` {#rpc_bind_addr} The address and port on which to bind for inter-cluster communcations (reffered to as RPC for remote procedure calls). @@ -282,14 +612,33 @@ the node, even in the case of a NAT: the NAT should be configured to forward the port number to the same internal port nubmer. This means that if you have several nodes running behind a NAT, they should each use a different RPC port number. -### `rpc_public_addr` +#### `rpc_bind_outgoing` (since `v0.9.2`) {#rpc_bind_outgoing} + +If enabled, pre-bind all sockets for outgoing connections to the same IP address +used for listening (the IP address specified in `rpc_bind_addr`) before +trying to connect to remote nodes. +This can be necessary if a node has multiple IP addresses, +but only one is allowed or able to reach the other nodes, +for instance due to firewall rules or specific routing configuration. + +Disabled by default. + +#### `rpc_public_addr` {#rpc_public_addr} The address and port that other nodes need to use to contact this node for RPC calls. **This parameter is optional but recommended.** In case you have a NAT that binds the RPC port to a port that is different on your public IP, this field might help making it work. -### `bootstrap_peers` +#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet} +In case `rpc_public_addr` is not set, but autodiscovery is used, this allows +filtering the list of automatically discovered IPs to a specific subnet. + +For example, if nodes should pick *their* IP inside a specific subnet, but you +don't want to explicitly write the IP down (as it's dynamic, or you want to +share configs across nodes), you can use this option. + +#### `bootstrap_peers` {#bootstrap_peers} A list of peer identifiers on which to contact other Garage peers of this cluster. These peer identifiers have the following syntax: @@ -305,75 +654,125 @@ be obtained by running `garage node id` and then included directly in the key will be returned by `garage node id` and you will have to add the IP yourself. +#### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets} -## The `[consul_discovery]` section +Garage checks the permissions of your secret files to make sure they're not +world-readable. In some cases, the check might fail and consider your files as +world-readable even if they're not, for instance when using Posix ACLs. + +Setting `allow_world_readable_secrets` to `true` bypass this +permission verification. + +Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS` +environment variable to `true` to bypass the permissions check. + +#### `allow_punycode` {#allow_punycode} + +Allow creating buckets with names containing punycode. When used for buckets served +as websites, this allows using almost any unicode character in the domain name. + +Default to `false`. + +### The `[consul_discovery]` section Garage supports discovering other nodes of the cluster using Consul. For this to work correctly, nodes need to know their IP address by which they can be reached by other nodes of the cluster, which should be set in `rpc_public_addr`. -### `consul_http_addr` and `service_name` +#### `consul_http_addr` {#consul_http_addr} The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server. -### `service_name` +#### `api` {#consul_api} + +Two APIs for service registration are supported: `catalog` and `agent`. `catalog`, the default, will register a service using +the `/v1/catalog` endpoints, enabling mTLS if `client_cert` and `client_key` are provided. The `agent` API uses the +`v1/agent` endpoints instead, where an optional `token` may be provided. + +#### `service_name` {#consul_service_name} `service_name` should be set to the service name under which Garage's RPC ports are announced. -### `client_cert`, `client_key` +#### `client_cert`, `client_key` {#consul_client_cert_and_key} TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so. +Only available when `api = "catalog"`. -### `ca_cert` +#### `ca_cert` {#consul_ca_cert} TLS CA certificate to use when communicating with Consul over TLS. -### `tls_skip_verify` +#### `tls_skip_verify` {#consul_tls_skip_verify} Skip server hostname verification in TLS handshake. `ca_cert` is ignored when this is set. +#### `token` {#consul_token} -## The `[kubernetes_discovery]` section +Uses the provided token for communication with Consul. Only available when `api = "agent"`. +The policy assigned to this token should at least have these rules: + +```hcl +// the `service_name` specified above +service "garage" { + policy = "write" +} + +service_prefix "" { + policy = "read" +} + +node_prefix "" { + policy = "read" +} +``` + +#### `tags` and `meta` {#consul_tags_and_meta} + +Additional list of tags and map of service meta to add during service registration. + +### The `[kubernetes_discovery]` section Garage supports discovering other nodes of the cluster using kubernetes custom resources. For this to work, a `[kubernetes_discovery]` section must be present with at least the `namespace` and `service_name` parameters. -### `namespace` +#### `namespace` {#kube_namespace} `namespace` sets the namespace in which the custom resources are configured. -### `service_name` +#### `service_name` {#kube_service_name} `service_name` is added as a label to the advertised resources to filter them, to allow for multiple deployments in a single namespace. -### `skip_crd` +#### `skip_crd` {#kube_skip_crd} `skip_crd` can be set to true to disable the automatic creation and patching of the `garagenodes.deuxfleurs.fr` CRD. You will need to create the CRD manually. -## The `[s3_api]` section +### The `[s3_api]` section -### `api_bind_addr` +#### `api_bind_addr` {#s3_api_bind_addr} The IP and port on which to bind for accepting S3 API calls. This endpoint does not suport TLS: a reverse proxy should be used to provide it. -### `s3_region` +Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode. + +#### `s3_region` {#s3_region} Garage will accept S3 API calls that are targetted to the S3 region defined here. API calls targetted to other regions will fail with a AuthorizationHeaderMalformed error message that redirects the client to the correct region. -### `root_domain` {#root_domain} +#### `root_domain` {#s3_root_domain} -The optionnal suffix to access bucket using vhost-style in addition to path-style request. +The optional suffix to access bucket using vhost-style in addition to path-style request. Note path-style requests are always enabled, whether or not vhost-style is configured. Configuring vhost-style S3 required a wildcard DNS entry, and possibly a wildcard TLS certificate, but might be required by softwares not supporting path-style requests. @@ -383,59 +782,73 @@ using the hostname `my-bucket.s3.garage.eu`. -## The `[s3_web]` section +### The `[s3_web]` section Garage allows to publish content of buckets as websites. This section configures the behaviour of this module. -### `bind_addr` +#### `bind_addr` {#web_bind_addr} The IP and port on which to bind for accepting HTTP requests to buckets configured for website access. This endpoint does not suport TLS: a reverse proxy should be used to provide it. -### `root_domain` +Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode. -The optionnal suffix appended to bucket names for the corresponding HTTP Host. +#### `root_domain` {#web_root_domain} + +The optional suffix appended to bucket names for the corresponding HTTP Host. For instance, if `root_domain` is `web.garage.eu`, a bucket called `deuxfleurs.fr` will be accessible either with hostname `deuxfleurs.fr.web.garage.eu` or with hostname `deuxfleurs.fr`. +#### `add_host_to_metrics` {#web_add_host_to_metrics} -## The `[admin]` section +Whether to include the requested domain name (HTTP `Host` header) in the +Prometheus metrics of the web endpoint. This is disabled by default as the +number of possible values is not bounded and can be a source of cardinality +explosion in the exported metrics. + + +### The `[admin]` section Garage has a few administration capabilities, in particular to allow remote monitoring. These features are detailed below. -### `api_bind_addr` +#### `api_bind_addr` {#admin_api_bind_addr} If specified, Garage will bind an HTTP server to this port and address, on which it will listen to requests for administration features. See [administration API reference](@/documentation/reference-manual/admin-api.md) to learn more about these features. -### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) +Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons, +the socket will have 0220 mode. Make sure to set user and group permissions accordingly. + +#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token} The token for accessing the Metrics endpoint. If this token is not set, the Metrics endpoint can be accessed without access control. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `metrics_token` was introduced in Garage `v0.7.2`. `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. -### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) +#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token} The token for accessing all of the other administration endpoints. If this token is not set, access to these endpoints is disabled entirely. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `admin_token` was introduced in Garage `v0.7.2`. `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. -### `trace_sink` +#### `trace_sink` {#admin_trace_sink} Optionally, the address of an OpenTelemetry collector. If specified, Garage will send traces in the OpenTelemetry format to this endpoint. These diff --git a/doc/book/reference-manual/features.md b/doc/book/reference-manual/features.md index 550504ff..481aef01 100644 --- a/doc/book/reference-manual/features.md +++ b/doc/book/reference-manual/features.md @@ -35,7 +35,22 @@ This makes setting up and administering storage clusters, we hope, as easy as it A Garage cluster can very easily evolve over time, as storage nodes are added or removed. Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies. -Read about cluster layout management [here](@/documentation/reference-manual/layout.md). +Read about cluster layout management [here](@/documentation/operations/layout.md). + +### Several replication modes + +Garage supports a variety of replication modes, with configurable replica count, +and with various levels of consistency, in order to adapt to a variety of usage scenarios. +Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_factor) +to select the replication mode best suited to your use case (hint: in most cases, `replication_factor = 3` is what you want). + +### Compression and deduplication + +All data stored in Garage is deduplicated, and optionnally compressed using +Zstd. Objects uploaded to Garage are chunked in blocks of constant sizes (see +[`block_size`](@/documentation/reference-manual/configuration.md#block_size)), +and the hashes of individual blocks are used to dispatch them to storage nodes +and to deduplicate them. ### No RAFT slowing you down @@ -46,14 +61,7 @@ directed to a Garage cluster can be handled independently of one another instead of going through a central bottleneck (the leader node). As a consequence, requests can be handled much faster, even in cases where latency between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this). -This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections. - -### Several replication modes - -Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data, -and with various levels of consistency, in order to adapt to a variety of usage scenarios. -Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode) -to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want). +This is particularly useful when nodes are far from one another and talk to one other through standard Internet connections. ### Web server for static websites diff --git a/doc/book/reference-manual/k2v.md b/doc/book/reference-manual/k2v.md index ed069b27..c01f641e 100644 --- a/doc/book/reference-manual/k2v.md +++ b/doc/book/reference-manual/k2v.md @@ -3,7 +3,7 @@ title = "K2V" weight = 100 +++ -Starting with version 0.7.2, Garage introduces an optionnal feature, K2V, +Starting with version 0.7.2, Garage introduces an optional feature, K2V, which is an alternative storage API designed to help efficiently store many small values in buckets (in opposition to S3 which is more designed to store large blobs). diff --git a/doc/book/reference-manual/layout.md b/doc/book/reference-manual/layout.md deleted file mode 100644 index a7d6f51f..00000000 --- a/doc/book/reference-manual/layout.md +++ /dev/null @@ -1,77 +0,0 @@ -+++ -title = "Cluster layout management" -weight = 50 -+++ - -The cluster layout in Garage is a table that assigns to each node a role in -the cluster. The role of a node in Garage can either be a storage node with -a certain capacity, or a gateway node that does not store data and is only -used as an API entry point for faster cluster access. -An introduction to building cluster layouts can be found in the [production deployment](@/documentation/cookbook/real-world.md) page. - -## How cluster layouts work in Garage - -In Garage, a cluster layout is composed of the following components: - -- a table of roles assigned to nodes -- a version number - -Garage nodes will always use the cluster layout with the highest version number. - -Garage nodes also maintain and synchronize between them a set of proposed role -changes that haven't yet been applied. These changes will be applied (or -canceled) in the next version of the layout - -The following commands insert modifications to the set of proposed role changes -for the next layout version (but they do not create the new layout immediately): - -```bash -garage layout assign [...] -garage layout remove [...] -``` - -The following command can be used to inspect the layout that is currently set in the cluster -and the changes proposed for the next layout version, if any: - -```bash -garage layout show -``` - -The following commands create a new layout with the specified version number, -that either takes into account the proposed changes or cancels them: - -```bash -garage layout apply --version -garage layout revert --version -``` - -The version number of the new layout to create must be 1 + the version number -of the previous layout that existed in the cluster. The `apply` and `revert` -commands will fail otherwise. - -## Warnings about Garage cluster layout management - -**Warning: never make several calls to `garage layout apply` or `garage layout -revert` with the same value of the `--version` flag. Doing so can lead to the -creation of several different layouts with the same version number, in which -case your Garage cluster will become inconsistent until fixed.** If a call to -`garage layout apply` or `garage layout revert` has failed and `garage layout -show` indicates that a new layout with the given version number has not been -set in the cluster, then it is fine to call the command again with the same -version number. - -If you are using the `garage` CLI by typing individual commands in your -shell, you shouldn't have much issues as long as you run commands one after -the other and take care of checking the output of `garage layout show` -before applying any changes. - -If you are using the `garage` CLI to script layout changes, follow the following recommendations: - -- Make all of your `garage` CLI calls to the same RPC host. Do not use the - `garage` CLI to connect to individual nodes to send them each a piece of the - layout changes you are making, as the changes propagate asynchronously - between nodes and might not all be taken into account at the time when the - new layout is applied. - -- **Only call `garage layout apply` once**, and call it **strictly after** all - of the `layout assign` and `layout remove` commands have returned. diff --git a/doc/book/reference-manual/monitoring.md b/doc/book/reference-manual/monitoring.md index 97c533d3..4696791e 100644 --- a/doc/book/reference-manual/monitoring.md +++ b/doc/book/reference-manual/monitoring.md @@ -27,6 +27,112 @@ Exposes the Garage replication factor configured on the node garage_replication_factor 3 ``` +#### `garage_local_disk_avail` and `garage_local_disk_total` (gauge) + +Reports the available and total disk space on each node, for data and metadata separately. + +``` +garage_local_disk_avail{volume="data"} 540341960704 +garage_local_disk_avail{volume="metadata"} 540341960704 +garage_local_disk_total{volume="data"} 763063566336 +garage_local_disk_total{volume="metadata"} 763063566336 +``` + +### Cluster health status metrics + +#### `cluster_healthy` (gauge) + +Whether all storage nodes are connected (0 or 1) + +``` +cluster_healthy 0 +``` + +#### `cluster_available` (gauge) + +Whether all requests can be served, even if some storage nodes are disconnected + +``` +cluster_available 1 +``` + +#### `cluster_connected_nodes` (gauge) + +Number of nodes currently connected + +``` +cluster_connected_nodes 3 +``` + +#### `cluster_known_nodes` (gauge) + +Number of nodes already seen once in the cluster + +``` +cluster_known_nodes 3 +``` + +#### `cluster_layout_node_connected` (gauge) + +Connection status for individual nodes of the cluster layout + +``` +cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +``` + +#### `cluster_layout_node_disconnected_time` (gauge) + +Time (in seconds) since last connection to individual nodes of the cluster layout + +``` +cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +``` + +#### `cluster_storage_nodes` (gauge) + +Number of storage nodes declared in the current layout + +``` +cluster_storage_nodes 4 +``` + +#### `cluster_storage_nodes_ok` (gauge) + +Number of storage nodes currently connected + +``` +cluster_storage_nodes_ok 3 +``` + +#### `cluster_partitions` (gauge) + +Number of partitions in the layout (this is always 256) + +``` +cluster_partitions 256 +``` + +#### `cluster_partitions_all_ok` (gauge) + +Number of partitions for which all storage nodes are connected + +``` +cluster_partitions_all_ok 64 +``` + +#### `cluster_partitions_quorum` (gauge) + +Number of partitions for which we have a quorum of connected nodes and all requests can be served + +``` +cluster_partitions_quorum 256 +``` + ### Metrics of the API endpoints #### `api_admin_request_counter` (counter) @@ -119,6 +225,17 @@ block_bytes_read 120586322022 block_bytes_written 3386618077 ``` +#### `block_ram_buffer_free_kb` (gauge) + +Kibibytes available for buffering blocks that have to be sent to remote nodes. +When clients send too much data to this node and a storage node is not receiving +data fast enough due to slower network conditions, this will decrease down to +zero and backpressure will be applied. + +``` +block_ram_buffer_free_kb 219829 +``` + #### `block_compression_level` (counter) Exposes the block compression level configured for the Garage node. @@ -275,7 +392,7 @@ table_merkle_updater_todo_queue_length{table_name="block_ref"} 0 #### `table_sync_items_received`, `table_sync_items_sent` (counters) -Number of data items sent to/recieved from other nodes during resync procedures +Number of data items sent to/received from other nodes during resync procedures ``` table_sync_items_received{from="",table_name="bucket_v2"} 3 diff --git a/doc/book/reference-manual/s3-compatibility.md b/doc/book/reference-manual/s3-compatibility.md index 15b29bd1..b869b6f4 100644 --- a/doc/book/reference-manual/s3-compatibility.md +++ b/doc/book/reference-manual/s3-compatibility.md @@ -23,16 +23,17 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them. - ## High-level features | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | +| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) | ✅ Implemented | ✅ | ✅ | ❌ | ✅ | | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) | ✅ Implemented | ✅ | ✅ | ❓| ✅ | | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) | +| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ | +| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ | *Note:* OpenIO does not says if it supports presigned URLs. Because it is part of signature v4 and they claim they support it without additional precisions, @@ -75,16 +76,13 @@ but these endpoints are documented in [Red Hat Ceph Storage - Chapter 2. Ceph Ob | Endpoint | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | -| [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) | ✅ Implemented (see details below) | ✅ | ✅ | ✅ | ✅ | -| [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html) | ✅ Implemented | ✅| ✅ | ✅ | ✅ | -| [ListMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUpload.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | -| [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | -| [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html) | ✅ Implemented (see details below) | ✅ | ✅| ✅ | ✅ | -| [UploadPartCopy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | - -Our implementation of Multipart Upload is currently a bit more restrictive than Amazon's one in some edge cases. -For more information, please refer to our [issue tracker](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/204). +| [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | +| [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | +| [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html) | ✅ Implemented | ✅| ✅ | ✅ | ✅ | +| [ListMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUpload.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | +| [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | +| [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html) | ✅ Implemented | ✅ | ✅| ✅ | ✅ | +| [UploadPartCopy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html) | ✅ Implemented | ✅ | ✅ | ✅ | ✅ | ### Website endpoints @@ -127,15 +125,22 @@ If you need this feature, please [share your use case in our dedicated issue](ht | Endpoint | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [DeleteBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html) | ❌ Missing | ❌| ✅| ❌| ✅| -| [GetBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html) | ❌ Missing | ❌| ✅ | ❌| ✅| -| [PutBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) | ❌ Missing | ❌| ✅ | ❌| ✅| +| [DeleteBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html) | ✅ Implemented | ❌| ✅| ❌| ✅| +| [GetBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html) | ✅ Implemented | ❌| ✅ | ❌| ✅| +| [PutBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) | ⚠ Partially implemented (see below) | ❌| ✅ | ❌| ✅| | [GetBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html) | ❌ Stub (see below) | ✅| ✅ | ❌| ✅| | [ListObjectVersions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html) | ❌ Missing | ❌| ✅ | ❌| ✅| | [PutBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) | ❌ Missing | ❌| ✅| ❌| ✅| +**PutBucketLifecycleConfiguration:** The only actions supported are +`AbortIncompleteMultipartUpload` and `Expiration` (without the +`ExpiredObjectDeleteMarker` field). All other operations are dependent on +either bucket versionning or storage classes which Garage currently does not +implement. The deprecated `Prefix` member directly in the the `Rule` +structure/XML tag is not supported, specified prefixes must be inside the +`Filter` structure/XML tag. -**GetBucketVersioning:** Stub implementation (Garage does not yet support versionning so this always returns "versionning not enabled"). +**GetBucketVersioning:** Stub implementation which always returns "versionning not enabled", since Garage does not yet support bucket versionning. ### Replication endpoints diff --git a/doc/book/working-documents/_index.md b/doc/book/working-documents/_index.md index 8fc170b7..fe79e65d 100644 --- a/doc/book/working-documents/_index.md +++ b/doc/book/working-documents/_index.md @@ -1,6 +1,6 @@ +++ title = "Working Documents" -weight = 8 +weight = 90 sort_by = "weight" template = "documentation.html" +++ diff --git a/doc/book/working-documents/design-draft.md b/doc/book/working-documents/design-draft.md index 6560dbed..8d3a31f0 100644 --- a/doc/book/working-documents/design-draft.md +++ b/doc/book/working-documents/design-draft.md @@ -42,7 +42,7 @@ The general principle are similar, but details have not been updated.** A version is defined by the existence of at least one entry in the blocks table for a certain version UUID. We must keep the following invariant: if a version exists in the blocks table, it has to be referenced in the objects table. We explicitly manage concurrent versions of an object: the version timestamp and version UUID columns are index columns, thus we may have several concurrent versions of an object. -Important: before deleting an older version from the objects table, we must make sure that we did a successfull delete of the blocks of that version from the blocks table. +Important: before deleting an older version from the objects table, we must make sure that we did a successful delete of the blocks of that version from the blocks table. Thus, the workflow for reading an object is as follows: @@ -95,7 +95,7 @@ Known issue: if someone is reading from a version that we want to delete and the Usefull metadata: - list of versions that reference this block in the Casandra table, so that we can do GC by checking in Cassandra that the lines still exist -- list of other nodes that we know have acknowledged a write of this block, usefull in the rebalancing algorithm +- list of other nodes that we know have acknowledged a write of this block, useful in the rebalancing algorithm Write strategy: have a single thread that does all write IO so that it is serialized (or have several threads that manage independent parts of the hash space). When writing a blob, write it to a temporary file, close, then rename so that a concurrent read gets a consistent result (either not found or found with whole content). diff --git a/doc/book/working-documents/migration-04.md b/doc/book/working-documents/migration-04.md index d9d3ede1..52c56737 100644 --- a/doc/book/working-documents/migration-04.md +++ b/doc/book/working-documents/migration-04.md @@ -68,7 +68,7 @@ The migration steps are as follows: 5. Turn off Garage 0.3 6. Backup metadata folders if you can (i.e. if you have space to do it - somewhere). Backuping data folders could also be usefull but that's much + somewhere). Backuping data folders could also be useful but that's much harder to do. If your filesystem supports snapshots, this could be a good time to use them. diff --git a/doc/book/working-documents/migration-09.md b/doc/book/working-documents/migration-09.md new file mode 100644 index 00000000..ba758093 --- /dev/null +++ b/doc/book/working-documents/migration-09.md @@ -0,0 +1,72 @@ ++++ +title = "Migrating from 0.8 to 0.9" +weight = 12 ++++ + +**This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster. +We don't recommend trying to migrate to 0.9 directly from 0.7 or older.** + +This migration procedure has been tested on several clusters without issues. +However, it is still a *critical procedure* that might cause issues. +**Make sure to back up all your data before attempting it!** + +You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md). + +The following are **breaking changes** in Garage v0.9 that require your attention when migrating: + +- LMDB is now the default metadata db engine and Sled is deprecated. If you were using Sled, make sure to specify `db_engine = "sled"` in your configuration file, or take the time to [convert your database](https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0). + +- Capacity values are now in actual byte units. The translation from the old layout will assign 1 capacity = 1Gb by default, which might be wrong for your cluster. This does not cause any data to be moved around, but you might want to re-assign correct capacity values post-migration. + +- Multipart uploads that were started in Garage v0.8 will not be visible in Garage v0.9 and will have to be restarted from scratch. + +- Changes to the admin API: some `v0/` endpoints have been replaced by `v1/` counterparts with updated/uniformized syntax. All other endpoints have also moved to `v1/` by default, without syntax changes, but are still available under `v0/` for compatibility. + + +## Simple migration procedure (takes cluster offline for a while) + +The migration steps are as follows: + +1. Disable API and web access. You may do this by stopping your reverse proxy or by commenting out + the `api_bind_addr` values in your `config.toml` file and restarting Garage. +2. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`, + check the logs and check that all data seems to be synced correctly between + nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.) +3. Check that the block resync queue and Merkle queue are empty: + run `garage stats -a` to query them or inspect metrics in the Grafana dashboard. +4. Turn off Garage v0.8 +5. **Backup the metadata folder of all your nodes!** For instance, use the following command + if your metadata directory is `/var/lib/garage/meta`: `cd /var/lib/garage ; tar -acf meta-v0.8.tar.zst meta/` +6. Install Garage v0.9 +7. Update your configuration file if necessary. +8. Turn on Garage v0.9 +9. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`. + Wait for a full table sync to run. +10. Your upgraded cluster should be in a working state. Re-enable API and Web + access and check that everything went well. +11. Monitor your cluster in the next hours to see if it works well under your production load, report any issue. +12. You might want to assign correct capacity values to all your nodes. Doing so might cause data to be moved + in your cluster, which should also be monitored carefully. + +## Minimal downtime migration procedure + +The migration to Garage v0.9 can be done with almost no downtime, +by restarting all nodes at once in the new version. + +The migration steps are as follows: + +1. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`, + check the logs and check that all data seems to be synced correctly between + nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.) + +2. Turn off each node individually; back up its metadata folder (see above); turn it back on again. + This will allow you to take a backup of all nodes without impacting global cluster availability. + You can do all nodes of a single zone at once as this does not impact the availability of Garage. + +3. Prepare your binaries and configuration files for Garage v0.9 + +4. Shut down all v0.8 nodes simultaneously, and restart them all simultaneously in v0.9. + Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to achieve this as fast as possible. + Garage v0.9 should be in a working state as soon as it starts. + +5. Proceed with repair and monitoring as described in steps 9-12 above. diff --git a/doc/book/working-documents/migration-1.md b/doc/book/working-documents/migration-1.md new file mode 100644 index 00000000..b6c0bb85 --- /dev/null +++ b/doc/book/working-documents/migration-1.md @@ -0,0 +1,77 @@ ++++ +title = "Migrating from 0.9 to 1.0" +weight = 11 ++++ + +**This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster. +We don't recommend trying to migrate to 1.0 directly from 0.8 or older.** + +This migration procedure has been tested on several clusters without issues. +However, it is still a *critical procedure* that might cause issues. +**Make sure to back up all your data before attempting it!** + +You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md). + +## Changes introduced in v1.0 + +The following are **breaking changes** in Garage v1.0 that require your attention when migrating: + +- The Sled metadata db engine has been **removed**. If your cluster was still + using Sled, you will need to **use a Garage v0.9.x binary** to convert the + database using the `garage convert-db` subcommand. See + [here](@/documentation/reference-manual/configuration.md#db_engine) for the + details of the procedure. + +The following syntax changes have been made to the configuration file: + +- The `replication_mode` parameter has been split into two parameters: + [`replication_factor`](@/documentation/reference-manual/configuration.md#replication_factor) + and + [`consistency_mode`](@/documentation/reference-manual/configuration.md#consistency_mode). + The old syntax using `replication_mode` is still supported for legacy + reasons and can still be used. + +- The parameters `sled_cache_capacity` and `sled_flush_every_ms` have been removed. + +## Migration procedure + +The migration to Garage v1.0 can be done with almost no downtime, +by restarting all nodes at once in the new version. + +The migration steps are as follows: + +1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that + all data seems to be synced correctly between nodes. If you have time, do + additional `garage repair` procedures (`blocks`, `versions`, `block_refs`, + etc.) + +2. Ensure you have a snapshot of your Garage installation that you can restore + to in case the upgrade goes wrong: + + - If you are running Garage v0.9.4 or later, use the `garage meta snapshot + --all` to make a backup snapshot of the metadata directories of your nodes + for backup purposes, and save a copy of the following files in the + metadata directories of your nodes: `cluster_layout`, `data_layout`, + `node_key`, `node_key.pub`. + + - If you are running a filesystem such as ZFS or BTRFS that support + snapshotting, you can create a filesystem-level snapshot to be used as a + restoration point if needed. + + - In other cases, make a backup using the old procedure: turn off each node + individually; back up its metadata folder (for instance, use the following + command if your metadata directory is `/var/lib/garage/meta`: `cd + /var/lib/garage ; tar -acf meta-v0.9.tar.zst meta/`); turn it back on + again. This will allow you to take a backup of all nodes without + impacting global cluster availability. You can do all nodes of a single + zone at once as this does not impact the availability of Garage. + +3. Prepare your updated binaries and configuration files for Garage v1.0 + +4. Shut down all v0.9 nodes simultaneously, and restart them all simultaneously + in v1.0. Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to + achieve this as fast as possible. Garage v1.0 should be in a working state + as soon as enough nodes have started. + +5. Monitor your cluster in the following hours to see if it works well under + your production load. diff --git a/doc/book/working-documents/testing-strategy.md b/doc/book/working-documents/testing-strategy.md index 7d6be8ef..fff706d7 100644 --- a/doc/book/working-documents/testing-strategy.md +++ b/doc/book/working-documents/testing-strategy.md @@ -37,7 +37,7 @@ There are two reasons for this: Reminder: rules of simplicity, concerning changes to Garage's source code. Always question what we are doing. -Never do anything just because it looks nice or because we "think" it might be usefull at some later point but without knowing precisely why/when. +Never do anything just because it looks nice or because we "think" it might be useful at some later point but without knowing precisely why/when. Only do things that make perfect sense in the context of what we currently know. ## References diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 9a697a59..3ee948cb 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -8,9 +8,9 @@ listen address is specified in the `[admin]` section of the configuration file (see [configuration file reference](@/documentation/reference-manual/configuration.md)) -**WARNING.** At this point, there is no comittement to stability of the APIs described in this document. -We will bump the version numbers prefixed to each API endpoint at each time the syntax -or semantics change, meaning that code that relies on these endpoint will break +**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document. +We will bump the version numbers prefixed to each API endpoint each time the syntax +or semantics change, meaning that code that relies on these endpoints will break when changes are introduced. The Garage administration API was introduced in version 0.7.2, this document @@ -19,7 +19,7 @@ does not apply to older versions of Garage. ## Access control -The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: +The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without @@ -52,11 +52,11 @@ Returns an HTTP status 200 if the node is ready to answer user's requests, and an HTTP status 503 (Service Unavailable) if there are some partitions for which a quorum of nodes is not available. A simple textual message is also returned in a body with content-type `text/plain`. -See `/v0/health` for an API that also returns JSON output. +See `/v1/health` for an API that also returns JSON output. ### Cluster operations -#### GetClusterStatus `GET /v0/status` +#### GetClusterStatus `GET /v1/status` Returns the cluster's current status in JSON, including: @@ -69,87 +69,121 @@ Example response body: ```json { - "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", - "garage_version": "git:v0.8.0", - "knownNodes": { - "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { - "addr": "10.0.0.11:3901", - "is_up": true, - "last_seen_secs_ago": 9, - "hostname": "node1" - }, - "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { - "addr": "10.0.0.12:3901", - "is_up": true, - "last_seen_secs_ago": 1, - "hostname": "node2" - }, - "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { - "addr": "10.0.0.21:3901", - "is_up": true, - "last_seen_secs_ago": 7, - "hostname": "node3" - }, - "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { - "addr": "10.0.0.22:3901", - "is_up": true, - "last_seen_secs_ago": 1, - "hostname": "node4" - } - }, - "layout": { - "version": 12, - "roles": { - "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { + "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "garageVersion": "v1.3.0", + "garageFeatures": [ + "k2v", + "lmdb", + "sqlite", + "metrics", + "bundled-libs" + ], + "rustVersion": "1.68.0", + "dbEngine": "LMDB (using Heed crate)", + "layoutVersion": 5, + "nodes": [ + { + "id": "62b218d848e86a64f7fe1909735f29a4350547b54c4b204f91246a14eb0a1a8c", + "role": { + "id": "62b218d848e86a64f7fe1909735f29a4350547b54c4b204f91246a14eb0a1a8c", "zone": "dc1", - "capacity": 4, - "tags": [ - "node1" - ] + "capacity": 100000000000, + "tags": [] }, - "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { - "zone": "dc1", - "capacity": 6, - "tags": [ - "node2" - ] + "addr": "10.0.0.3:3901", + "hostname": "node3", + "isUp": true, + "lastSeenSecsAgo": 12, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 }, - "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { - "zone": "dc2", - "capacity": 10, - "tags": [ - "node3" - ] + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 } }, - "stagedRoleChanges": { - "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { - "zone": "dc2", - "capacity": 5, - "tags": [ - "node4" - ] + { + "id": "a11c7cf18af297379eff8688360155fe68d9061654449ba0ce239252f5a7487f", + "role": null, + "addr": "10.0.0.2:3901", + "hostname": "node2", + "isUp": true, + "lastSeenSecsAgo": 11, + "draining": true, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } + }, + { + "id": "a235ac7695e0c54d7b403943025f57504d500fdcc5c3e42c71c5212faca040a2", + "role": { + "id": "a235ac7695e0c54d7b403943025f57504d500fdcc5c3e42c71c5212faca040a2", + "zone": "dc1", + "capacity": 100000000000, + "tags": [] + }, + "addr": "127.0.0.1:3904", + "hostname": "lindy", + "isUp": true, + "lastSeenSecsAgo": 2, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 + } + }, + { + "id": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "role": { + "id": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", + "zone": "dc1", + "capacity": 100000000000, + "tags": [] + }, + "addr": "10.0.0.1:3901", + "hostname": "node1", + "isUp": true, + "lastSeenSecsAgo": 3, + "draining": false, + "dataPartition": { + "available": 660270088192, + "total": 873862266880 + }, + "metadataPartition": { + "available": 660270088192, + "total": 873862266880 } } - } + ] } ``` -#### GetClusterHealth `GET /v0/health` +#### GetClusterHealth `GET /v1/health` Returns the cluster's current health in JSON format, with the following variables: -- `status`: one of `Healthy`, `Degraded` or `Unavailable`: - - Healthy: Garage node is connected to all storage nodes - - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions - - Unavailable: a quorum of write nodes is not available for some partitions -- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started -- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to -- `storage_nodes`: the number of storage nodes currently registered in the cluster layout -- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open +- `status`: one of `healthy`, `degraded` or `unavailable`: + - healthy: Garage node is connected to all storage nodes + - degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions + - unavailable: a quorum of write nodes is not available for some partitions +- `knownNodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started +- `connectedNodes`: the nubmer of nodes this Garage node currently has an open connection to +- `storageNodes`: the number of storage nodes currently registered in the cluster layout +- `storageNodesOk`: the number of storage nodes to which a connection is currently open - `partitions`: the total number of partitions of the data (currently always 256) -- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available -- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it +- `partitionsQuorum`: the number of partitions for which a quorum of write nodes is available +- `partitionsAllOk`: the number of partitions for which we are connected to all storage nodes responsible of storing it Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code. @@ -157,18 +191,18 @@ Example response body: ```json { - "status": "Degraded", - "known_nodes": 3, - "connected_nodes": 2, - "storage_nodes": 3, - "storage_nodes_ok": 2, - "partitions": 256, - "partitions_quorum": 256, - "partitions_all_ok": 0 + "status": "degraded", + "knownNodes": 3, + "connectedNodes": 3, + "storageNodes": 4, + "storageNodesOk": 3, + "partitions": 256, + "partitionsQuorum": 256, + "partitionsAllOk": 64 } ``` -#### ConnectClusterNodes `POST /v0/connect` +#### ConnectClusterNodes `POST /v1/connect` Instructs this Garage node to connect to other Garage nodes at specified addresses. @@ -198,7 +232,7 @@ Example response: ] ``` -#### GetClusterLayout `GET /v0/layout` +#### GetClusterLayout `GET /v1/layout` Returns the cluster's current layout in JSON, including: @@ -212,42 +246,54 @@ Example response body: ```json { "version": 12, - "roles": { - "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { + "roles": [ + { + "id": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", "zone": "dc1", - "capacity": 4, + "capacity": 10737418240, "tags": [ "node1" ] }, - "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { + { + "id": "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff", "zone": "dc1", - "capacity": 6, + "capacity": 10737418240, "tags": [ "node2" ] }, - "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { + { + "id": "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27", "zone": "dc2", - "capacity": 10, + "capacity": 10737418240, "tags": [ "node3" ] } - }, - "stagedRoleChanges": { - "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { + ], + "stagedRoleChanges": [ + { + "id": "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b", + "remove": false, "zone": "dc2", - "capacity": 5, + "capacity": 10737418240, "tags": [ "node4" ] } - } + { + "id": "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27", + "remove": true, + "zone": null, + "capacity": null, + "tags": null, + } + ] } ``` -#### UpdateClusterLayout `POST /v0/layout` +#### UpdateClusterLayout `POST /v1/layout` Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls @@ -259,8 +305,9 @@ the layout. Request body format: ```json -{ - : { +[ + { + "id": , "capacity": , "zone": , "tags": [ @@ -268,17 +315,22 @@ Request body format: ... ] }, - : null, - ... -} + { + "id": , + "remove": true + } +] ``` Contrary to the CLI that may update only a subset of the fields `capacity`, `zone` and `tags`, when calling this API all of these values must be specified. +This returns the new cluster layout with the proposed staged changes, +as returned by GetClusterLayout. -#### ApplyClusterLayout `POST /v0/layout/apply` + +#### ApplyClusterLayout `POST /v1/layout/apply` Applies to the cluster the layout changes currently registered as staged layout changes. @@ -295,7 +347,10 @@ Similarly to the CLI, the body must include the version of the new layout that will be created, which MUST be 1 + the value of the currently existing layout in the cluster. -#### RevertClusterLayout `POST /v0/layout/revert` +This returns the message describing all the calculations done to compute the new +layout, as well as the description of the layout as returned by GetClusterLayout. + +#### RevertClusterLayout `POST /v1/layout/revert` Clears all of the staged layout changes. @@ -313,10 +368,13 @@ Similarly to the CLI, the body must include the incremented version number, which MUST be 1 + the value of the currently existing layout in the cluster. +This returns the new cluster layout with all changes reverted, +as returned by GetClusterLayout. + ### Access key operations -#### ListKeys `GET /v0/key` +#### ListKeys `GET /v1/key` Returns all API access keys in the cluster. @@ -335,34 +393,8 @@ Example response: ] ``` -#### CreateKey `POST /v0/key` - -Creates a new API access key. - -Request body format: - -```json -{ - "name": "NameOfMyKey" -} -``` - -#### ImportKey `POST /v0/key/import` - -Imports an existing API key. - -Request body format: - -```json -{ - "accessKeyId": "GK31c2f218a2e44f485b94239e", - "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835", - "name": "NameOfMyKey" -} -``` - -#### GetKeyInfo `GET /v0/key?id=` -#### GetKeyInfo `GET /v0/key?search=` +#### GetKeyInfo `GET /v1/key?id=` +#### GetKeyInfo `GET /v1/key?search=` Returns information about the requested API access key. @@ -370,6 +402,9 @@ If `id` is set, the key is looked up using its exact identifier (faster). If `search` is set, the key is looked up using its name or prefix of identifier (slower, all keys are enumerated to do this). +Optionnally, the query parameter `showSecretKey=true` can be set to reveal the +associated secret access key. + Example response: ```json @@ -433,11 +468,40 @@ Example response: } ``` -#### DeleteKey `DELETE /v0/key?id=` +#### CreateKey `POST /v1/key` -Deletes an API access key. +Creates a new API access key. -#### UpdateKey `POST /v0/key?id=` +Request body format: + +```json +{ + "name": "NameOfMyKey" +} +``` + +This returns the key info, including the created secret key, +in the same format as the result of GetKeyInfo. + +#### ImportKey `POST /v1/key/import` + +Imports an existing API key. +This will check that the imported key is in the valid format, i.e. +is a key that could have been generated by Garage. + +Request body format: + +```json +{ + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835", + "name": "NameOfMyKey" +} +``` + +This returns the key info in the same format as the result of GetKeyInfo. + +#### UpdateKey `POST /v1/key?id=` Updates information about the specified API access key. @@ -453,14 +517,20 @@ Request body format: } ``` -All fields (`name`, `allow` and `deny`) are optionnal. +All fields (`name`, `allow` and `deny`) are optional. If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed. The possible flags in `allow` and `deny` are: `createBucket`. +This returns the key info in the same format as the result of GetKeyInfo. + +#### DeleteKey `DELETE /v1/key?id=` + +Deletes an API access key. + ### Bucket operations -#### ListBuckets `GET /v0/bucket` +#### ListBuckets `GET /v1/bucket` Returns all storage buckets in the cluster. @@ -502,8 +572,8 @@ Example response: ] ``` -#### GetBucketInfo `GET /v0/bucket?id=` -#### GetBucketInfo `GET /v0/bucket?globalAlias=` +#### GetBucketInfo `GET /v1/bucket?id=` +#### GetBucketInfo `GET /v1/bucket?globalAlias=` Returns information about the requested storage bucket. @@ -535,7 +605,10 @@ Example response: ], "objects": 14827, "bytes": 13189855625, - "unfinshedUploads": 0, + "unfinishedUploads": 1, + "unfinishedMultipartUploads": 1, + "unfinishedMultipartUploadParts": 11, + "unfinishedMultipartUploadBytes": 41943040, "quotas": { "maxSize": null, "maxObjects": null @@ -543,7 +616,7 @@ Example response: } ``` -#### CreateBucket `POST /v0/bucket` +#### CreateBucket `POST /v1/bucket` Creates a new storage bucket. @@ -583,13 +656,7 @@ or no alias at all. Technically, you can also specify both `globalAlias` and `localAlias` and that would create two aliases, but I don't see why you would want to do that. -#### DeleteBucket `DELETE /v0/bucket?id=` - -Deletes a storage bucket. A bucket cannot be deleted if it is not empty. - -Warning: this will delete all aliases associated with the bucket! - -#### UpdateBucket `PUT /v0/bucket?id=` +#### UpdateBucket `PUT /v1/bucket?id=` Updates configuration of the given bucket. @@ -609,7 +676,7 @@ Request body format: } ``` -All fields (`websiteAccess` and `quotas`) are optionnal. +All fields (`websiteAccess` and `quotas`) are optional. If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. @@ -621,9 +688,16 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or to remove the quotas. An absent value will be considered the same as a `null`. It is not possible to change only one of the two quotas. +#### DeleteBucket `DELETE /v1/bucket?id=` + +Deletes a storage bucket. A bucket cannot be deleted if it is not empty. + +Warning: this will delete all aliases associated with the bucket! + + ### Operations on permissions for keys on buckets -#### BucketAllowKey `POST /v0/bucket/allow` +#### BucketAllowKey `POST /v1/bucket/allow` Allows a key to do read/write/owner operations on a bucket. @@ -644,7 +718,7 @@ Request body format: Flags in `permissions` which have the value `true` will be activated. Other flags will remain unchanged. -#### BucketDenyKey `POST /v0/bucket/deny` +#### BucketDenyKey `POST /v1/bucket/deny` Denies a key from doing read/write/owner operations on a bucket. @@ -668,19 +742,19 @@ Other flags will remain unchanged. ### Operations on bucket aliases -#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=&alias=` +#### GlobalAliasBucket `PUT /v1/bucket/alias/global?id=&alias=` Empty body. Creates a global alias for a bucket. -#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=&alias=` +#### GlobalUnaliasBucket `DELETE /v1/bucket/alias/global?id=&alias=` Removes a global alias for a bucket. -#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=&accessKeyId=&alias=` +#### LocalAliasBucket `PUT /v1/bucket/alias/local?id=&accessKeyId=&alias=` Empty body. Creates a local alias for a bucket in the namespace of a specific access key. -#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=&accessKeyId&alias=` +#### LocalUnaliasBucket `DELETE /v1/bucket/alias/local?id=&accessKeyId&alias=` Removes a local alias for a bucket in the namespace of a specific access key. diff --git a/doc/drafts/k2v-spec.md b/doc/drafts/k2v-spec.md index faa1a247..f9696717 100644 --- a/doc/drafts/k2v-spec.md +++ b/doc/drafts/k2v-spec.md @@ -146,7 +146,7 @@ in a bucket, as the partition key becomes the sort key in the index. How indexing works: - Each node keeps a local count of how many items it stores for each partition, - in a local Sled tree that is updated atomically when an item is modified. + in a local database tree that is updated atomically when an item is modified. - These local counters are asynchronously stored in the index table which is a regular Garage table spread in the network. Counters are stored as LWW values, so basically the final table will have the following structure: @@ -562,7 +562,7 @@ token>", v: ["", ...] }`, with the following fields: - in case of concurrent update and deletion, a `null` is added to the list of concurrent values - if the `tombstones` query parameter is set to `true`, tombstones are returned - for items that have been deleted (this can be usefull for inserting after an + for items that have been deleted (this can be useful for inserting after an item that has been deleted, so that the insert is not considered concurrent with the delete). Tombstones are returned as tuples in the same format with only `null` values diff --git a/doc/optimal_layout_report/.gitignore b/doc/optimal_layout_report/.gitignore new file mode 100644 index 00000000..d5e59136 --- /dev/null +++ b/doc/optimal_layout_report/.gitignore @@ -0,0 +1,13 @@ +optimal_layout.aux +optimal_layout.log +optimal_layout.synctex.gz +optimal_layout.bbl +optimal_layout.blg + +geodistrib.aux +geodistrib.bbl +geodistrib.blg +geodistrib.log +geodistrib.out +geodistrib.synctex.gz + diff --git a/doc/optimal_layout_report/figures/flow.pdf b/doc/optimal_layout_report/figures/flow.pdf new file mode 100644 index 00000000..3546ad0a Binary files /dev/null and b/doc/optimal_layout_report/figures/flow.pdf differ diff --git a/doc/optimal_layout_report/figures/flow.svg b/doc/optimal_layout_report/figures/flow.svg new file mode 100644 index 00000000..e370755e --- /dev/null +++ b/doc/optimal_layout_report/figures/flow.svg @@ -0,0 +1,2205 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/optimal_layout_report/figures/flow_graph_param.pdf b/doc/optimal_layout_report/figures/flow_graph_param.pdf new file mode 100644 index 00000000..25b1205a Binary files /dev/null and b/doc/optimal_layout_report/figures/flow_graph_param.pdf differ diff --git a/doc/optimal_layout_report/figures/flow_graph_param.svg b/doc/optimal_layout_report/figures/flow_graph_param.svg new file mode 100644 index 00000000..1ef27ec5 --- /dev/null +++ b/doc/optimal_layout_report/figures/flow_graph_param.svg @@ -0,0 +1,7817 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/optimal_layout_report/figures/mini_node.pdf b/doc/optimal_layout_report/figures/mini_node.pdf new file mode 100644 index 00000000..6df8a5b2 Binary files /dev/null and b/doc/optimal_layout_report/figures/mini_node.pdf differ diff --git a/doc/optimal_layout_report/figures/mini_node.svg b/doc/optimal_layout_report/figures/mini_node.svg new file mode 100644 index 00000000..b044b0cd --- /dev/null +++ b/doc/optimal_layout_report/figures/mini_node.svg @@ -0,0 +1,3962 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/optimal_layout_report/figures/mini_zone.pdf b/doc/optimal_layout_report/figures/mini_zone.pdf new file mode 100644 index 00000000..36085c52 Binary files /dev/null and b/doc/optimal_layout_report/figures/mini_zone.pdf differ diff --git a/doc/optimal_layout_report/figures/mini_zone.svg b/doc/optimal_layout_report/figures/mini_zone.svg new file mode 100644 index 00000000..5c505539 --- /dev/null +++ b/doc/optimal_layout_report/figures/mini_zone.svg @@ -0,0 +1,1562 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/optimal_layout_report/figures/naive.pdf b/doc/optimal_layout_report/figures/naive.pdf new file mode 100644 index 00000000..f32e4273 Binary files /dev/null and b/doc/optimal_layout_report/figures/naive.pdf differ diff --git a/doc/optimal_layout_report/figures/naive.svg b/doc/optimal_layout_report/figures/naive.svg new file mode 100644 index 00000000..0a40c45f --- /dev/null +++ b/doc/optimal_layout_report/figures/naive.svg @@ -0,0 +1,3899 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/optimal_layout_report/geodistrib.pdf b/doc/optimal_layout_report/geodistrib.pdf new file mode 100644 index 00000000..68269a09 Binary files /dev/null and b/doc/optimal_layout_report/geodistrib.pdf differ diff --git a/doc/optimal_layout_report/geodistrib.tex b/doc/optimal_layout_report/geodistrib.tex new file mode 100644 index 00000000..bb6f0391 --- /dev/null +++ b/doc/optimal_layout_report/geodistrib.tex @@ -0,0 +1,317 @@ +\documentclass[]{article} + +\usepackage{amsmath,amssymb} +\usepackage{amsthm} + +\usepackage{stmaryrd} + +\usepackage{graphicx,xcolor} +\usepackage{hyperref} + +\usepackage{algorithm,algpseudocode,float} + +\renewcommand\thesubsubsection{\Alph{subsubsection})} + +\newtheorem{proposition}{Proposition} + +%opening +\title{An algorithm for geo-distributed and redundant storage in Garage} +\author{Mendes Oulamara \\ \emph{mendes@deuxfleurs.fr}} +\date{} + +\begin{document} + +\maketitle + +\begin{abstract} + Garage +\end{abstract} + +\section{Introduction} + +Garage\footnote{\url{https://garagehq.deuxfleurs.fr/}} is an open-source distributed object storage service tailored for self-hosting. It was designed by the Deuxfleurs association\footnote{\url{https://deuxfleurs.fr/}} to enable small structures (associations, collectives, small companies) to share storage resources to reliably self-host their data, possibly with old and non-reliable machines. + +To achieve these reliability and availability goals, the data is broken into \emph{partitions} and every partition is replicated over 3 different machines (that we call \emph{nodes}). When the data is queried, a consensus algorithm allows to fetch it from one of the nodes. A \emph{replication factor} of 3 ensures the best guarantees in the consensus algorithm \cite{ADD RREF}, but this parameter can be different. + +Moreover, if the nodes are spread over different \emph{zones} (different houses, offices, cities\dots), we can ask the data to be replicated over nodes belonging to different zones, to improve the storage robustness against zone failure (such as power outage). To do so, we set a \emph{redundancy parameter}, that is no more than the replication factor, and we ask that any partition is replicated over this number of zones at least. + +In this work, we propose a repartition algorithm that, given the nodes specifications and the replication and redundancy parameters, computes an optimal assignation of partitions to nodes. We say that the assignation is optimal in the sense that it maximizes the size of the partitions, and hence the effective storage capacity of the system. + +Moreover, when a former assignation exists, which is not optimal anymore due to nodes or zones updates, our algorithm computes a new optimal assignation that minimizes the amount of data to be transferred during the assignation update (the \emph{transfer load}). + +We call the set of nodes cooperating to store the data a \emph{cluster}, and a description of the nodes, zones and the assignation of partitions to nodes a \emph{cluster layout} + +\subsection{Notations} + +Let $k$ be some fixed parameter value, typically 8, that we call the ``partition bits''. +Every object to be stored in the system is split into data blocks of fixed size. We compute a hash $h(\mathbf{b})$ of every such block $\mathbf{b}$, and we define the $k$ last bits of this hash to be the partition number $p(\mathbf{b})$ of the block. This label can take $P=2^k$ different values, and hence there are $P$ different partitions. We denote $\mathbf{P}$ the set of partition labels (i.e. $\mathbf{P}=\llbracket1,P\rrbracket$). + +We are given a set $\mathbf{N}$ of $N$ nodes and a set $\mathbf{Z}$ of $Z$ zones. Every node $n$ has a non-negative storage capacity $c_n\ge 0$ and belongs to a zone $z_n\in \mathbf{Z}$. We are also given a replication parameter $\rho_\mathbf{N}$ and a redundancy parameter $\rho_\mathbf{Z}$ such that $1\le \rho_\mathbf{Z} \le \rho_\mathbf{N}$ (typical values would be $\rho_N=3$ and $\rho_Z=2$). + +Our goal is to compute an assignment $\alpha = (\alpha_p^1, \ldots, \alpha_p^{\rho_\mathbf{N}})_{p\in \mathbf{P}}$ such that every partition $p$ is associated to $\rho_\mathbf{N}$ distinct nodes $\alpha_p^1, \ldots, \alpha_p^{\rho_\mathbf{N}} \in \mathbf{N}$ and these nodes belong to at least $\rho_\mathbf{Z}$ distinct zones. Among the possible assignations, we choose one that \emph{maximizes} the effective storage capacity of the cluster. If the layout contained a previous assignment $\alpha'$, we \emph{minimize} the amount of data to transfer during the layout update by making $\alpha$ as close as possible to $\alpha'$. These maximization and minimization are described more formally in the following section. + +\subsection{Optimization parameters} + +To link the effective storage capacity of the cluster to partition assignment, we make the following assumption: +\begin{equation} + \tag{H1} + \text{\emph{All partitions have the same size $s$.}} +\end{equation} +This assumption is justified by the dispersion of the hashing function, when the number of partitions is small relative to the number of stored blocks. + +Every node $n$ wille store some number $p_n$ of partitions (it is the number of partitions $p$ such that $n$ appears in the $\alpha_p$). Hence the partitions stored by $n$ (and hence all partitions by our assumption) have there size bounded by $c_n/p_n$. This remark leads us to define the optimal size that we will want to maximize: + +\begin{equation} + \label{eq:optimal} + \tag{OPT} +s^* = \min_{n \in N} \frac{c_n}{p_n}. +\end{equation} + +When the capacities of the nodes are updated (this includes adding or removing a node), we want to update the assignment as well. However, transferring the data between nodes has a cost and we would like to limit the number of changes in the assignment. We make the following assumption: +\begin{equation} + \tag{H2} + \text{\emph{Nodes updates happen rarely relatively to block operations.}} +\end{equation} +This assumption justifies that when we compute the new assignment $\alpha$, it is worth to optimize the partition size \eqref{eq:optimal} first, and then, among the possible optimal solution, to try to minimize the number of partition transfers. More formally, we minimize the distance between two assignments defined by +\begin{equation} + d(\alpha, \alpha') := \#\{ (n,p) \in \mathbf{N}\times\mathbf{P} ~|~ n\in \alpha_p \triangle \alpha'_p \} +\end{equation} +where the symmetric difference $\alpha_p \triangle \alpha'_p$ denotes the nodes appearing in one of the assignations but not in both. + +\section{Computation of an optimal assignment} + +The algorithm that we propose takes as inputs the cluster layout parameters $\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$, that we defined in the introduction, together with the former assignation $\alpha'$ (if any). The computation of the new optimal assignation $\alpha^*$ is done in three successive steps that will be detailed in the following sections. The first step computes the largest partition size $s^*$ that an assignation can achieve. The second step computes an optimal candidate assignment $\alpha$ that achieves $s^*$ and a heuristic is used in the computation to make it hopefully close to $\alpha'$. The third steps modifies $\alpha$ iteratively to reduces $d(\alpha, \alpha')$ and yields an assignation $\alpha^*$ achieving $s^*$, and minimizing $d(\cdot, \alpha')$ among such assignations. + +We will explain in the next section how to represent an assignment $\alpha$ by a flow $f$ on a weighted graph $G$ to enable the use of flow and graph algorithms. The main function of the algorithm can be written as follows. + +\subsubsection*{Algorithm} + +\begin{algorithmic}[1] + \Function{Compute Layout}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$, $\alpha'$} + \State $s^* \leftarrow$ \Call{Compute Partition Size}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$} + \State $G \leftarrow G(s^*)$ + \State $f \leftarrow$ \Call{Compute Candidate Assignment}{$G$, $\alpha'$} + \State $f^* \leftarrow$ \Call{Minimize transfer load}{$G$, $f$, $\alpha'$} + \State Build $\alpha^*$ from $f^*$ + \State \Return $\alpha^*$ + \EndFunction +\end{algorithmic} + +\subsubsection*{Complexity} +As we will see in the next sections, the worst case complexity of this algorithm is $O(P^2 N^2)$. The minimization of transfer load is the most expensive step, and it can run with a timeout since it is only an optimization step. Without this step (or with a smart timeout), the worst cas complexity can be $O((PN)^{3/2}\log C)$ where $C$ is the total storage capacity of the cluster. + +\subsection{Determination of the partition size $s^*$} + +We will represent an assignment $\alpha$ as a flow in a specific graph $G$. We will not compute the optimal partition size $s^*$ a priori, but we will determine it by dichotomy, as the largest size $s$ such that the maximal flow achievable on $G=G(s)$ has value $\rho_\mathbf{N}P$. We will assume that the capacities are given in a small enough unit (say, Megabytes), and we will determine $s^*$ at the precision of the given unit. + +Given some candidate size value $s$, we describe the oriented weighted graph $G=(V,E)$ with vertex set $V$ arc set $E$ (see Figure \ref{fig:flowgraph}). + +The set of vertices $V$ contains the source $\mathbf{s}$, the sink $\mathbf{t}$, vertices +$\mathbf{p^+, p^-}$ for every partition $p$, vertices $\mathbf{x}_{p,z}$ for every partition $p$ and zone $z$, and vertices $\mathbf{n}$ for every node $n$. + +The set of arcs $E$ contains: +\begin{itemize} + \item ($\mathbf{s}$,$\mathbf{p}^+$, $\rho_\mathbf{Z}$) for every partition $p$; + \item ($\mathbf{s}$,$\mathbf{p}^-$, $\rho_\mathbf{N}-\rho_\mathbf{Z}$) for every partition $p$; + \item ($\mathbf{p}^+$,$\mathbf{x}_{p,z}$, 1) for every partition $p$ and zone $z$; + \item ($\mathbf{p}^-$,$\mathbf{x}_{p,z}$, $\rho_\mathbf{N}-\rho_\mathbf{Z}$) for every partition $p$ and zone $z$; + \item ($\mathbf{x}_{p,z}$,$\mathbf{n}$, 1) for every partition $p$, zone $z$ and node $n\in z$; + \item ($\mathbf{n}$, $\mathbf{t}$, $\lfloor c_n/s \rfloor$) for every node $n$. +\end{itemize} + +\begin{figure} + \centering + \includegraphics[width=\linewidth]{figures/flow_graph_param} + \caption{An example of graph $G(s)$. Arcs are oriented from left to right, and unlabeled arcs have capacity 1. In this example, nodes $n_1,n_2,n_3$ belong to zone $z_1$, and nodes $n_4,n_5$ belong to zone $z_2$.} + \label{fig:flowgraph} +\end{figure} + +In the following complexity calculations, we will use the number of vertices and edges of $G$. Remark from now that $\# V = O(PZ)$ and $\# E = O(PN)$. + +\begin{proposition} + An assignment $\alpha$ is realizable with partition size $s$ and the redundancy constraints $(\rho_\mathbf{N},\rho_\mathbf{Z})$ if and only if there exists a maximal flow function $f$ in $G$ with total flow $\rho_\mathbf{N}P$, such that the arcs ($\mathbf{x}_{p,z}$,$\mathbf{n}$, 1) used are exactly those for which $p$ is associated to $n$ in $\alpha$. +\end{proposition} +\begin{proof} + Given such flow $f$, we can reconstruct a candidate $\alpha$. In $f$, the flow passing through $\mathbf{p^+}$ and $\mathbf{p^-}$ is $\rho_\mathbf{N}$, and since the outgoing capacity of every $\mathbf{x}_{p,z}$ is 1, every partition is associated to $\rho_\mathbf{N}$ distinct nodes. The fraction $\rho_\mathbf{Z}$ of the flow passing through every $\mathbf{p^+}$ must be spread over as many distinct zones as every arc outgoing from $\mathbf{p^+}$ has capacity 1. So the reconstructed $\alpha$ verifies the redundancy constraints. For every node $n$, the flow between $\mathbf{n}$ and $\mathbf{t}$ corresponds to the number of partitions associated to $n$. By construction of $f$, this does not exceed $\lfloor c_n/s \rfloor$. We assumed that the partition size is $s$, hence this association does not exceed the storage capacity of the nodes. + + In the other direction, given an assignment $\alpha$, one can similarly check that the facts that $\alpha$ respects the redundancy constraints, and the storage capacities of the nodes, are necessary condition to construct a maximal flow function $f$. +\end{proof} + +\textbf{Implementation remark:} In the flow algorithm, while exploring the graph, we explore the neighbours of every vertex in a random order to heuristically spread the associations between nodes and partitions. + +\subsubsection*{Algorithm} +With this result mind, we can describe the first step of our algorithm. All divisions are supposed to be integer divisions. +\begin{algorithmic}[1] + \Function{Compute Partition Size}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$} + + \State Build the graph $G=G(s=1)$ + \State $ f \leftarrow$ \Call{Maximal flow}{$G$} + \If{$f.\mathrm{total flow} < \rho_\mathbf{N}P$} + + \State \Return Error: capacities too small or constraints too strong. + \EndIf + + \State $s^- \leftarrow 1$ + \State $s^+ \leftarrow 1+\frac{1}{\rho_\mathbf{N}}\sum_{n \in \mathbf{N}} c_n$ + + \While{$s^-+1 < s^+$} + \State Build the graph $G=G(s=(s^-+s^+)/2)$ + \State $ f \leftarrow$ \Call{Maximal flow}{$G$} + \If{$f.\mathrm{total flow} < \rho_\mathbf{N}P$} + \State $s^+ \leftarrow (s^- + s^+)/2$ + \Else + \State $s^- \leftarrow (s^- + s^+)/2$ + \EndIf + \EndWhile + + \State \Return $s^-$ + \EndFunction +\end{algorithmic} + +\subsubsection*{Complexity} + +To compute the maximal flow, we use Dinic's algorithm. Its complexity on general graphs is $O(\#V^2 \#E)$, but on graphs with edge capacity bounded by a constant, it turns out to be $O(\#E^{3/2})$. The graph $G$ does not fall in this case since the capacities of the arcs incoming to $\mathbf{t}$ are far from bounded. However, the proof of this complexity function works readily for graphs where we only ask the edges \emph{not} incoming to the sink $\mathbf{t}$ to have their capacities bounded by a constant. One can find the proof of this claim in \cite[Section 2]{even1975network}. +The dichotomy adds a logarithmic factor $\log (C)$ where $C=\sum_{n \in \mathbf{N}} c_n$ is the total capacity of the cluster. The total complexity of this first function is hence +$O(\#E^{3/2}\log C ) = O\big((PN)^{3/2} \log C\big)$. + +\subsubsection*{Metrics} +We can display the discrepancy between the computed $s^*$ and the best size we could have hoped for the given total capacity, that is $C/\rho_\mathbf{N}$. + +\subsection{Computation of a candidate assignment} + +Now that we have the optimal partition size $s^*$, to compute a candidate assignment it would be enough to compute a maximal flow function $f$ on $G(s^*)$. This is what we do if there is no former assignation $\alpha'$. + +If there is some $\alpha'$, we add a step that will heuristically help to obtain a candidate $\alpha$ closer to $\alpha'$. We fist compute a flow function $\tilde{f}$ that uses only the partition-to-node associations appearing in $\alpha'$. Most likely, $\tilde{f}$ will not be a maximal flow of $G(s^*)$. In Dinic's algorithm, we can start from a non maximal flow function and then discover improving paths. This is what we do by starting from $\tilde{f}$. The hope\footnote{This is only a hope, because one can find examples where the construction of $f$ from $\tilde{f}$ produces an assignment $\alpha$ that is not as close as possible to $\alpha'$.} is that the final flow function $f$ will tend to keep the associations appearing in $\tilde{f}$. + +More formally, we construct the graph $G_{|\alpha'}$ from $G$ by removing all the arcs $(\mathbf{x}_{p,z},\mathbf{n}, 1)$ where $p$ is not associated to $n$ in $\alpha'$. We compute a maximal flow function $\tilde{f}$ in $G_{|\alpha'}$. The flow $\tilde{f}$ is also a valid (most likely non maximal) flow function on $G$. We compute a maximal flow function $f$ on $G$ by starting Dinic's algorithm on $\tilde{f}$. + +\subsubsection*{Algorithm} +\begin{algorithmic}[1] + \Function{Compute Candidate Assignment}{$G$, $\alpha'$} + \State Build the graph $G_{|\alpha'}$ + \State $ \tilde{f} \leftarrow$ \Call{Maximal flow}{$G_{|\alpha'}$} + \State $ f \leftarrow$ \Call{Maximal flow from flow}{$G$, $\tilde{f}$} + \State \Return $f$ + \EndFunction +\end{algorithmic} + +~ + +\textbf{Remark:} The function ``Maximal flow'' can be just seen as the function ``Maximal flow from flow'' called with the zero flow function as starting flow. + +\subsubsection*{Complexity} +With the considerations of the last section, we have the complexity of the Dinic's algorithm $O(\#E^{3/2}) = O((PN)^{3/2})$. + +\subsubsection*{Metrics} + +We can display the flow value of $\tilde{f}$, which is an upper bound of the distance between $\alpha$ and $\alpha'$. It might be more a Debug level display than Info. + +\subsection{Minimization of the transfer load} + +Now that we have a candidate flow function $f$, we want to modify it to make its corresponding assignation $\alpha$ as close as possible to $\alpha'$. Denote by $f'$ the maximal flow corresponding to $\alpha'$, and let $d(f, \alpha')=d(f, f'):=d(\alpha,\alpha')$\footnote{It is the number of arcs of type $(\mathbf{x}_{p,z},\mathbf{n})$ saturated in one flow and not in the other.}. +We want to build a sequence $f=f_0, f_1, f_2 \dots$ of maximal flows such that $d(f_i, \alpha')$ decreases as $i$ increases. The distance being a non-negative integer, this sequence of flow functions must be finite. We now explain how to find some improving $f_{i+1}$ from $f_i$. + +For any maximal flow $f$ in $G$, we define the oriented weighted graph $G_f=(V, E_f)$ as follows. The vertices of $G_f$ are the same as the vertices of $G$. $E_f$ contains the arc $(v_1,v_2, w)$ between vertices $v_1,v_2\in V$ with weight $w$ if and only if the arc $(v_1,v_2)$ is not saturated in $f$ (i.e. $c(v_1,v_2)-f(v_1,v_2) \ge 1$, we also consider reversed arcs). The weight $w$ is: +\begin{itemize} + \item $-1$ if $(v_1,v_2)$ is of type $(\mathbf{x}_{p,z},\mathbf{n})$ or $(\mathbf{x}_{p,z},\mathbf{n})$ and is saturated in only one of the two flows $f,f'$; + \item $+1$ if $(v_1,v_2)$ is of type $(\mathbf{x}_{p,z},\mathbf{n})$ or $(\mathbf{x}_{p,z},\mathbf{n})$ and is saturated in either both or none of the two flows $f,f'$; + \item $0$ otherwise. +\end{itemize} + +If $\gamma$ is a simple cycle of arcs in $G_f$, we define its weight $w(\gamma)$ as the sum of the weights of its arcs. We can add $+1$ to the value of $f$ on the arcs of $\gamma$, and by construction of $G_f$ and the fact that $\gamma$ is a cycle, the function that we get is still a valid flow function on $G$, it is maximal as it has the same flow value as $f$. We denote this new function $f+\gamma$. + +\begin{proposition} + Given a maximal flow $f$ and a simple cycle $\gamma$ in $G_f$, we have $d(f+\gamma, f') - d(f,f') = w(\gamma)$. +\end{proposition} +\begin{proof} + Let $X$ be the set of arcs of type $(\mathbf{x}_{p,z},\mathbf{n})$. Then we can express $d(f,f')$ as + \begin{align*} + d(f,f') & = \#\{e\in X ~|~ f(e)\neq f'(e)\} + = \sum_{e\in X} 1_{f(e)\neq f'(e)} \\ + & = \frac{1}{2}\big( \#X + \sum_{e\in X} 1_{f(e)\neq f'(e)} - 1_{f(e)= f'(e)} \big). + \end{align*} + We can express the cycle weight as + \begin{align*} + w(\gamma) & = \sum_{e\in X, e\in \gamma} - 1_{f(e)\neq f'(e)} + 1_{f(e)= f'(e)}. + \end{align*} + Remark that since we passed on unit of flow in $\gamma$ to construct $f+\gamma$, we have for any $e\in X$, $f(e)=f'(e)$ if and only if $(f+\gamma)(e) \neq f'(e)$. + Hence + \begin{align*} + w(\gamma) & = \frac{1}{2}(w(\gamma) + w(\gamma)) \\ + &= \frac{1}{2} \Big( + \sum_{e\in X, e\in \gamma} - 1_{f(e)\neq f'(e)} + 1_{f(e)= f'(e)} \\ + & \qquad + + \sum_{e\in X, e\in \gamma} 1_{(f+\gamma)(e)\neq f'(e)} + 1_{(f+\gamma)(e)= f'(e)} + \Big). + \end{align*} + Plugging this in the previous equation, we find that + $$d(f,f')+w(\gamma) = d(f+\gamma, f').$$ +\end{proof} + +This result suggests that given some flow $f_i$, we just need to find a negative cycle $\gamma$ in $G_{f_i}$ to construct $f_{i+1}$ as $f_i+\gamma$. The following proposition ensures that this greedy strategy reaches an optimal flow. + +\begin{proposition} + For any maximal flow $f$, $G_f$ contains a negative cycle if and only if there exists a maximal flow $f^*$ in $G$ such that $d(f^*, f') < d(f, f')$. +\end{proposition} +\begin{proof} + Suppose that there is such flow $f^*$. Define the oriented multigraph $M_{f,f^*}=(V,E_M)$ with the same vertex set $V$ as in $G$, and for every $v_1,v_2 \in V$, $E_M$ contains $(f^*(v_1,v_2) - f(v_1,v_2))_+$ copies of the arc $(v_1,v_2)$. For every vertex $v$, its total degree (meaning its outer degree minus its inner degree) is equal to + \begin{align*} + \deg v & = \sum_{u\in V} (f^*(v,u) - f(v,u))_+ - \sum_{u\in V} (f^*(u,v) - f(u,v))_+ \\ + & = \sum_{u\in V} f^*(v,u) - f(v,u) = \sum_{u\in V} f^*(v,u) - \sum_{u\in V} f(v,u). + \end{align*} + The last two sums are zero for any inner vertex since $f,f^*$ are flows, and they are equal on the source and sink since the two flows are both maximal and have hence the same value. Thus, $\deg v = 0$ for every vertex $v$. + + This implies that the multigraph $M_{f,f^*}$ is the union of disjoint simple cycles. $f$ can be transformed into $f^*$ by pushing a mass 1 along all these cycles in any order. Since $d(f^*, f')0$ (typically $P=256$). + +We would like to compute an assignment of nodes to partitions. We will impose some redundancy constraints to this assignment, and under these constraints, we want our system to have the largest storage capacity possible. To link storage capacity to partition assignment, we make the following assumption: +\begin{equation} + \tag{H1} + \text{\emph{All partitions have the same size $s$.}} +\end{equation} +This assumption is justified by the dispersion of the hashing function, when the number of partitions is small relative to the number of stored large objects. + +Every node $n$ wille store some number $k_n$ of partitions. Hence the partitions stored by $n$ (and hence all partitions by our assumption) have there size bounded by $c_n/k_n$. This remark leads us to define the optimal size that we will want to maximize: + +\begin{equation} + \label{eq:optimal} + \tag{OPT} +s^* = \min_{n \in N} \frac{c_n}{k_n}. +\end{equation} + +When the capacities of the nodes are updated (this includes adding or removing a node), we want to update the assignment as well. However, transferring the data between nodes has a cost and we would like to limit the number of changes in the assignment. We make the following assumption: +\begin{equation} + \tag{H2} + \text{\emph{Updates of capacity happens rarely relatively to object storing.}} +\end{equation} +This assumption justifies that when we compute the new assignment, it is worth to optimize the partition size \eqref{eq:optimal} first, and then, among the possible optimal solution, to try to minimize the number of partition transfers. + +For now, in the following, we ask the following redundancy constraint: + +\textbf{Parametric node and zone redundancy:} Given two integer parameters $1\le \rho_\mathbf{Z} \le \rho_\mathbf{N}$, we ask every partition to be stored on $\rho_\mathbf{N}$ distinct nodes, and these nodes must belong to at least $\rho_\mathbf{Z}$ distinct zones. + + +\textbf{Mode 3-strict:} every partition needs to be assignated to three nodes belonging to three different zones. + +\textbf{Mode 3:} every partition needs to be assignated to three nodes. We try to spread the three nodes over different zones as much as possible. + +\textbf{Warning:} This is a working document written incrementaly. The last version of the algorithm is the \textbf{parametric assignment} described in the next section. + + +\section{Computation of a parametric assignment} +\textbf{Attention : }We change notations in this section. + +Notations : let $P$ be the number of partitions, $N$ the number of nodes, $Z$ the number of zones. Let $\mathbf{P,N,Z}$ be the label sets of, respectively, partitions, nodes and zones. +Let $s^*$ be the largest partition size achievable with the redundancy constraints. Let $(c_n)_{n\in \mathbf{N}}$ be the storage capacity of every node. + +In this section, we propose a third specification of the problem. The user inputs two redundancy parameters $1\le \rho_\mathbf{Z} \le \rho_\mathbf{N}$. We compute an assignment $\alpha = (\alpha_p^1, \ldots, \alpha_p^{\rho_\mathbf{N}})_{p\in \mathbf{P}}$ such that every partition $p$ is associated to $\rho_\mathbf{N}$ distinct nodes $\alpha_p^1, \ldots, \alpha_p^{\rho_\mathbf{N}}$ and these nodes belong to at least $\rho_\mathbf{Z}$ distinct zones. + +If the layout contained a previous assignment $\alpha'$, we try to minimize the amount of data to transfer during the layout update by making $\alpha$ as close as possible to $\alpha'$. + +In the following subsections, we describe the successive steps of the algorithm we propose to compute $\alpha$. + +\subsubsection*{Algorithm} + +\begin{algorithmic}[1] + \Function{Compute Layout}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$, $\alpha'$} + \State $s^* \leftarrow$ \Call{Compute Partition Size}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$} + \State $G \leftarrow G(s^*)$ + \State $f \leftarrow$ \Call{Compute Candidate Assignment}{$G$, $\alpha'$} + \State $f^* \leftarrow$ \Call{Minimize transfer load}{$G$, $f$, $\alpha'$} + \State Build $\alpha^*$ from $f^*$ + \State \Return $\alpha^*$ + \EndFunction +\end{algorithmic} + +\subsubsection*{Complexity} +As we will see in the next sections, the worst case complexity of this algorithm is $O(P^2 N^2)$. The minimization of transfer load is the most expensive step, and it can run with a timeout since it is only an optimization step. Without this step (or with a smart timeout), the worst cas complexity can be $O((PN)^{3/2}\log C)$ where $C$ is the total storage capacity of the cluster. + +\subsection{Determination of the partition size $s^*$} + +Again, we will represent an assignment $\alpha$ as a flow in a specific graph $G$. We will not compute the optimal partition size $s^*$ a priori, but we will determine it by dichotomy, as the largest size $s$ such that the maximal flow achievable on $G=G(s)$ has value $\rho_\mathbf{N}P$. We will assume that the capacities are given in a small enough unit (say, Megabytes), and we will determine $s^*$ at the precision of the given unit. + +Given some candidate size value $s$, we describe the oriented weighted graph $G=(V,E)$ with vertex set $V$ arc set $E$. + +The set of vertices $V$ contains the source $\mathbf{s}$, the sink $\mathbf{t}$, vertices +$\mathbf{p^+, p^-}$ for every partition $p$, vertices $\mathbf{x}_{p,z}$ for every partition $p$ and zone $z$, and vertices $\mathbf{n}$ for every node $n$. + +The set of arcs $E$ contains: +\begin{itemize} + \item ($\mathbf{s}$,$\mathbf{p}^+$, $\rho_\mathbf{Z}$) for every partition $p$; + \item ($\mathbf{s}$,$\mathbf{p}^-$, $\rho_\mathbf{N}-\rho_\mathbf{Z}$) for every partition $p$; + \item ($\mathbf{p}^+$,$\mathbf{x}_{p,z}$, 1) for every partition $p$ and zone $z$; + \item ($\mathbf{p}^-$,$\mathbf{x}_{p,z}$, $\rho_\mathbf{N}-\rho_\mathbf{Z}$) for every partition $p$ and zone $z$; + \item ($\mathbf{x}_{p,z}$,$\mathbf{n}$, 1) for every partition $p$, zone $z$ and node $n\in z$; + \item ($\mathbf{n}$, $\mathbf{t}$, $\lfloor c_n/s \rfloor$) for every node $n$. +\end{itemize} + +In the following complexity calculations, we will use the number of vertices and edges of $G$. Remark from now that $\# V = O(PZ)$ and $\# E = O(PN)$. + +\begin{proposition} + An assignment $\alpha$ is realizable with partition size $s$ and the redundancy constraints $(\rho_\mathbf{N},\rho_\mathbf{Z})$ if and only if there exists a maximal flow function $f$ in $G$ with total flow $\rho_\mathbf{N}P$, such that the arcs ($\mathbf{x}_{p,z}$,$\mathbf{n}$, 1) used are exactly those for which $p$ is associated to $n$ in $\alpha$. +\end{proposition} +\begin{proof} + Given such flow $f$, we can reconstruct a candidate $\alpha$. In $f$, the flow passing through $\mathbf{p^+}$ and $\mathbf{p^-}$ is $\rho_\mathbf{N}$, and since the outgoing capacity of every $\mathbf{x}_{p,z}$ is 1, every partition is associated to $\rho_\mathbf{N}$ distinct nodes. The fraction $\rho_\mathbf{Z}$ of the flow passing through every $\mathbf{p^+}$ must be spread over as many distinct zones as every arc outgoing from $\mathbf{p^+}$ has capacity 1. So the reconstructed $\alpha$ verifies the redundancy constraints. For every node $n$, the flow between $\mathbf{n}$ and $\mathbf{t}$ corresponds to the number of partitions associated to $n$. By construction of $f$, this does not exceed $\lfloor c_n/s \rfloor$. We assumed that the partition size is $s$, hence this association does not exceed the storage capacity of the nodes. + + In the other direction, given an assignment $\alpha$, one can similarly check that the facts that $\alpha$ respects the redundancy constraints, and the storage capacities of the nodes, are necessary condition to construct a maximal flow function $f$. +\end{proof} + +\textbf{Implementation remark:} In the flow algorithm, while exploring the graph, we explore the neighbours of every vertex in a random order to heuristically spread the association between nodes and partitions. + +\subsubsection*{Algorithm} +With this result mind, we can describe the first step of our algorithm. All divisions are supposed to be integer division. +\begin{algorithmic}[1] + \Function{Compute Partition Size}{$\mathbf{N}$, $\mathbf{Z}$, $\mathbf{P}$, $(c_n)_{n\in \mathbf{N}}$, $\rho_\mathbf{N}$, $\rho_\mathbf{Z}$} + + \State Build the graph $G=G(s=1)$ + \State $ f \leftarrow$ \Call{Maximal flow}{$G$} + \If{$f.\mathrm{total flow} < \rho_\mathbf{N}P$} + + \State \Return Error: capacities too small or constraints too strong. + \EndIf + + \State $s^- \leftarrow 1$ + \State $s^+ \leftarrow 1+\frac{1}{\rho_\mathbf{N}}\sum_{n \in \mathbf{N}} c_n$ + + \While{$s^-+1 < s^+$} + \State Build the graph $G=G(s=(s^-+s^+)/2)$ + \State $ f \leftarrow$ \Call{Maximal flow}{$G$} + \If{$f.\mathrm{total flow} < \rho_\mathbf{N}P$} + \State $s^+ \leftarrow (s^- + s^+)/2$ + \Else + \State $s^- \leftarrow (s^- + s^+)/2$ + \EndIf + \EndWhile + + \State \Return $s^-$ + \EndFunction +\end{algorithmic} + +\subsubsection*{Complexity} + +To compute the maximal flow, we use Dinic's algorithm. Its complexity on general graphs is $O(\#V^2 \#E)$, but on graphs with edge capacity bounded by a constant, it turns out to be $O(\#E^{3/2})$. The graph $G$ does not fall in this case since the capacities of the arcs incoming to $\mathbf{t}$ are far from bounded. However, the proof of this complexity works readily for graph where we only ask the edges \emph{not} incoming to the sink $\mathbf{t}$ to have their capacities bounded by a constant. One can find the proof of this claim in \cite[Section 2]{even1975network}. +The dichotomy adds a logarithmic factor $\log (C)$ where $C=\sum_{n \in \mathbf{N}} c_n$ is the total capacity of the cluster. The total complexity of this first function is hence +$O(\#E^{3/2}\log C ) = O\big((PN)^{3/2} \log C\big)$. + +\subsubsection*{Metrics} +We can display the discrepancy between the computed $s^*$ and the best size we could hope for a given total capacity, that is $C/\rho_\mathbf{N}$. + +\subsection{Computation of a candidate assignment} + +Now that we have the optimal partition size $s^*$, to compute a candidate assignment, it would be enough to compute a maximal flow function $f$ on $G(s^*)$. This is what we do if there was no previous assignment $\alpha'$. + +If there was some $\alpha'$, we add a step that will heuristically help to obtain a candidate $\alpha$ closer to $\alpha'$. to do so, we fist compute a flow function $\tilde{f}$ that uses only the partition-to-node association appearing in $\alpha'$. Most likely, $\tilde{f}$ will not be a maximal flow of $G(s^*)$. In Dinic's algorithm, we can start from a non maximal flow function and then discover improving paths. This is what we do in starting from $\tilde{f}$. The hope\footnote{This is only a hope, because one can find examples where the construction of $f$ from $\tilde{f}$ produces an assignment $\alpha$ that is not as close as possible to $\alpha'$.} is that the final flow function $f$ will tend to keep the associations appearing in $\tilde{f}$. + +More formally, we construct the graph $G_{|\alpha'}$ from $G$ by removing all the arcs $(\mathbf{x}_{p,z},\mathbf{n}, 1)$ where $p$ is not associated to $n$ in $\alpha'$. We compute a maximal flow function $\tilde{f}$ in $G_{|\alpha'}$. $\tilde{f}$ is also a valid (most likely non maximal) flow function in $G$. We compute a maximal flow function $f$ on $G$ by starting Dinic's algorithm on $\tilde{f}$. + +\subsubsection*{Algorithm} +\begin{algorithmic}[1] + \Function{Compute Candidate Assignment}{$G$, $\alpha'$} + \State Build the graph $G_{|\alpha'}$ + \State $ \tilde{f} \leftarrow$ \Call{Maximal flow}{$G_{|\alpha'}$} + \State $ f \leftarrow$ \Call{Maximal flow from flow}{$G$, $\tilde{f}$} + \State \Return $f$ + \EndFunction +\end{algorithmic} + +\textbf{Remark:} The function ``Maximal flow'' can be just seen as the function ``Maximal flow from flow'' called with the zero flow function as starting flow. + +\subsubsection*{Complexity} +From the consideration of the last section, we have the complexity of the Dinic's algorithm $O(\#E^{3/2}) = O((PN)^{3/2})$. + +\subsubsection*{Metrics} + +We can display the flow value of $\tilde{f}$, which is an upper bound of the distance between $\alpha$ and $\alpha'$. It might be more a Debug level display than Info. + +\subsection{Minimization of the transfer load} + +Now that we have a candidate flow function $f$, we want to modify it to make its associated assignment as close as possible to $\alpha'$. Denote by $f'$ the maximal flow associated to $\alpha'$, and let $d(f, f')$ be distance between the associated assignments\footnote{It is the number of arcs of type $(\mathbf{x}_{p,z},\mathbf{n})$ saturated in one flow and not in the other.}. +We want to build a sequence $f=f_0, f_1, f_2 \dots$ of maximal flows such that $d(f_i, \alpha')$ decreases as $i$ increases. The distance being a non-negative integer, this sequence of flow functions must be finite. We now explain how to find some improving $f_{i+1}$ from $f_i$. + +For any maximal flow $f$ in $G$, we define the oriented weighted graph $G_f=(V, E_f)$ as follows. The vertices of $G_f$ are the same as the vertices of $G$. $E_f$ contains the arc $(v_1,v_2, w)$ between vertices $v_1,v_2\in V$ with weight $w$ if and only if the arc $(v_1,v_2)$ is not saturated in $f$ (i.e. $c(v_1,v_2)-f(v_1,v_2) \ge 1$, we also consider reversed arcs). The weight $w$ is: +\begin{itemize} + \item $-1$ if $(v_1,v_2)$ is of type $(\mathbf{x}_{p,z},\mathbf{n})$ or $(\mathbf{x}_{p,z},\mathbf{n})$ and is saturated in only one of the two flows $f,f'$; + \item $+1$ if $(v_1,v_2)$ is of type $(\mathbf{x}_{p,z},\mathbf{n})$ or $(\mathbf{x}_{p,z},\mathbf{n})$ and is saturated in either both or none of the two flows $f,f'$; + \item $0$ otherwise. +\end{itemize} + +If $\gamma$ is a simple cycle of arcs in $G_f$, we define its weight $w(\gamma)$ as the sum of the weights of its arcs. We can add $+1$ to the value of $f$ on the arcs of $\gamma$, and by construction of $G_f$ and the fact that $\gamma$ is a cycle, the function that we get is still a valid flow function on $G$, it is maximal as it has the same flow value as $f$. We denote this new function $f+\gamma$. + +\begin{proposition} + Given a maximal flow $f$ and a simple cycle $\gamma$ in $G_f$, we have $d(f+\gamma, f') - d(f,f') = w(\gamma)$. +\end{proposition} +\begin{proof} + Let $X$ be the set of arcs of type $(\mathbf{x}_{p,z},\mathbf{n})$. Then we can express $d(f,f')$ as + \begin{align*} + d(f,f') & = \#\{e\in X ~|~ f(e)\neq f'(e)\} + = \sum_{e\in X} 1_{f(e)\neq f'(e)} \\ + & = \frac{1}{2}\big( \#X + \sum_{e\in X} 1_{f(e)\neq f'(e)} - 1_{f(e)= f'(e)} \big). + \end{align*} + We can express the cycle weight as + \begin{align*} + w(\gamma) & = \sum_{e\in X, e\in \gamma} - 1_{f(e)\neq f'(e)} + 1_{f(e)= f'(e)}. + \end{align*} + Remark that since we passed on unit of flow in $\gamma$ to construct $f+\gamma$, we have for any $e\in X$, $f(e)=f'(e)$ if and only if $(f+\gamma)(e) \neq f'(e)$. + Hence + \begin{align*} + w(\gamma) & = \frac{1}{2}(w(\gamma) + w(\gamma)) \\ + &= \frac{1}{2} \Big( + \sum_{e\in X, e\in \gamma} - 1_{f(e)\neq f'(e)} + 1_{f(e)= f'(e)} \\ + & \qquad + + \sum_{e\in X, e\in \gamma} 1_{(f+\gamma)(e)\neq f'(e)} + 1_{(f+\gamma)(e)= f'(e)} + \Big). + \end{align*} + Plugging this in the previous equation, we find that + $$d(f,f')+w(\gamma) = d(f+\gamma, f').$$ +\end{proof} + +This result suggests that given some flow $f_i$, we just need to find a negative cycle $\gamma$ in $G_{f_i}$ to construct $f_{i+1}$ as $f_i+\gamma$. The following proposition ensures that this greedy strategy reaches an optimal flow. + +\begin{proposition} + For any maximal flow $f$, $G_f$ contains a negative cycle if and only if there exists a maximal flow $f^*$ in $G$ such that $d(f^*, f') < d(f, f')$. +\end{proposition} +\begin{proof} + Suppose that there is such flow $f^*$. Define the oriented multigraph $M_{f,f^*}=(V,E_M)$ with the same vertex set $V$ as in $G$, and for every $v_1,v_2 \in V$, $E_M$ contains $(f^*(v_1,v_2) - f(v_1,v_2))_+$ copies of the arc $(v_1,v_2)$. For every vertex $v$, its total degree (meaning its outer degree minus its inner degree) is equal to + \begin{align*} + \deg v & = \sum_{u\in V} (f^*(v,u) - f(v,u))_+ - \sum_{u\in V} (f^*(u,v) - f(u,v))_+ \\ + & = \sum_{u\in V} f^*(v,u) - f(v,u) = \sum_{u\in V} f^*(v,u) - \sum_{u\in V} f(v,u). + \end{align*} + The last two sums are zero for any inner vertex since $f,f^*$ are flows, and they are equal on the source and sink since the two flows are both maximal and have hence the same value. Thus, $\deg v = 0$ for every vertex $v$. + + This implies that the multigraph $M_{f,f^*}$ is the union of disjoint simple cycles. $f$ can be transformed into $f^*$ by pushing a mass 1 along all these cycles in any order. Since $d(f^*, f') n_v$, it means that all the partitions that could stay in $v$ (i.e. that were already in $v$ and are still assigned to its zone) do stay in $v$. If $\#P_v = n_v$, then $n_v$ partitions stay in $v$, which is the number of partitions that need to be in $v$ in the end. In both cases, we could not hope for better given the partition to zone assignment. + +Our goal now is to find a assignment of partitions to zones that minimizes the number of zone transfers. To do so we are going to represent an assignment as a graph. + +Let $G_T=(X,E_T)$ be the directed weighted graph with vertices $(\mathbf{x}_i)_{1\le i\le N}$ and $(\mathbf{y}_z)_{z\in Z}$. For any $1\le i\le N$ and $z\in Z$, $E_T$ contains the arc: +\begin{itemize} + \item $(\mathbf{x}_i, \mathbf{y}_z, +1)$, if $z$ appears in $T_i'$ and $T_i$; + \item $(\mathbf{x}_i, \mathbf{y}_z, -1)$, if $z$ appears in $T_i$ but not in $T'_i$; + \item $(\mathbf{y}_z, \mathbf{x}_i, -1)$, if $z$ appears in $T'_i$ but not in $T_i$; + \item $(\mathbf{y}_z, \mathbf{x}_i, +1)$, if $z$ does not appear in $T'_i$ nor in $T_i$. +\end{itemize} +In other words, the orientation of the arc encodes whether partition $i$ is stored in zone $z$ in the assignment $T$ and the weight $\pm 1$ encodes whether this corresponds to what happens in the assignment $T'$. + +\begin{figure}[t] + \centering + \begin{minipage}{.40\linewidth} + \centering + \includegraphics[width=.8\linewidth]{figures/mini_zone} + \end{minipage} + \begin{minipage}{.55\linewidth} + \centering + \includegraphics[width=.8\linewidth]{figures/mini_node} + \end{minipage} + \caption{On the left: the graph $G_T$ encoding an assignment to minimize the zone discrepancy. On the right: the graph $G_T$ encoding an assignment to minimize the node discrepancy.} +\end{figure} + + +Notice that at every partition, there are three outgoing arcs, and at every zone, there are $n_z$ incoming arcs. Moreover, if $w(e)$ is the weight of an arc $e$, define the weight of $G_T$ by +\begin{align*} +w(G_T) := \sum_{e\in E} w(e) &= \#Z \times N - 4 \sum_{1\le i\le N} \#\{z\in Z ~|~ z\cap T_i = \emptyset, z\cap T'_i \neq \emptyset\} \\ +&=\#Z \times N - 4 \sum_{1\le i\le N} 3- \#\{z\in Z ~|~ z\cap T_i \neq \emptyset, z\cap T'_i \neq \emptyset\} \\ +&= (\#Z-12)N + 4 Q_Z. +\end{align*} +Hence maximizing $Q_Z$ is equivalent to maximizing $w(G_T)$. + +Assume that their exist some assignment $T^*$ with the same utilization $(n_v)_{v\in V}$. Define $G_{T^*}$ similarly and consider the set $E_\mathrm{Diff} = E_T \setminus E_{T^*}$ of arcs that appear only in $G_T$. Since all vertices have the same number of incoming arcs in $G_T$ and $G_{T^*}$, the vertices of the graph $(X, E_\mathrm{Diff})$ must all have the same number number of incoming and outgoing arrows. So $E_\mathrm{Diff}$ can be expressed as a union of disjoint cycles. Moreover, the edges of $E_\mathrm{Diff}$ must appear in $E_{T^*}$ with reversed orientation and opposite weight. Hence, we have +$$ + w(G_T) - w(G_{T^*}) = 2 \sum_{e\in E_\mathrm{Diff}} w(e). +$$ +Hence, if $T$ is not optimal, there exists some $T^*$ with $w(G_T) < w(G_{T^*})$, and by the considerations above, there must exist a cycle in $E_\mathrm{Diff}$, and hence in $G_T$, with negative weight. If we reverse the edges and weights along this cycle, we obtain some graph. Since we did not change the incoming degree of any vertex, this is the graph encoding of some valid assignment $T^+$ such that $w(G_{T^+}) > w(G_T)$. We can iterate this operation until there is no other assignment $T^*$ with larger weight, that is until we obtain an optimal assignment. + + + +\subsubsection{Minimizing the node discrepancy} + +We will follow an approach similar to the one where we minimize the zone discrepancy. Here we will directly obtain a node assignment from a graph encoding. + +Let $G_T=(X,E_T)$ be the directed weighted graph with vertices $(\mathbf{x}_i)_{1\le i\le N}$, $(\mathbf{y}_{z,i})_{z\in Z, 1\le i\le N}$ and $(\mathbf{u}_v)_{v\in V}$. For any $1\le i\le N$ and $z\in Z$, $E_T$ contains the arc: +\begin{itemize} + \item $(\mathbf{x}_i, \mathbf{y}_{z,i}, 0)$, if $z$ appears in $T_i$; + \item $(\mathbf{y}_{z,i}, \mathbf{x}_i, 0)$, if $z$ does not appear in $T_i$. +\end{itemize} +For any $1\le i\le N$ and $v\in V$, $E_T$ contains the arc: +\begin{itemize} + \item $(\mathbf{y}_{z_v,i}, \mathbf{u}_v, +1)$, if $v$ appears in $T_i'$ and $T_i$; + \item $(\mathbf{y}_{z_v,i}, \mathbf{u}_v, -1)$, if $v$ appears in $T_i$ but not in $T'_i$; + \item $(\mathbf{u}_v, \mathbf{y}_{z_v,i}, -1)$, if $v$ appears in $T'_i$ but not in $T_i$; + \item $(\mathbf{u}_v, \mathbf{y}_{z_v,i}, +1)$, if $v$ does not appear in $T'_i$ nor in $T_i$. +\end{itemize} +Every vertex $\mathbb{x}_i$ has outgoing degree 3, every vertex $\mathbf{y}_{z,v}$ has outgoing degree 1, and every vertex $\mathbf{u}_v$ has incoming degree $n_v$. +Remark that any graph respecting these degree constraints is the encoding of a valid assignment with utilizations $(n_v)_{v\in V}$, in particular no partition is stored in two nodes of the same zone. + +We define $w(G_T)$ similarly: +\begin{align*} + w(G_T) := \sum_{e\in E_T} w(e) &= \#V \times N - 4\sum_{1\le i\le N} 3-\#(T_i\cap T'_i) \\ + &= (\#V-12)N + 4Q_V. +\end{align*} + +Exactly like in the previous section, the existence of an assignment with larger weight implies the existence of a negatively weighted cycle in $G_T$. Reversing this cycle gives us the encoding of a valid assignment with a larger weight. Iterating this operation yields an optimal assignment. + + +\subsubsection{Linear combination of both criteria} + +In the graph $G_T$ defined in the previous section, instead of having weights $0$ and $\pm 1$, we could be having weights $\pm\alpha$ between $\mathbf{x}$ and $\mathbf{y}$ vertices, and weights $\pm\beta$ between $\mathbf{y}$ and $\mathbf{u}$ vertices, for some $\alpha,\beta>0$ (we have positive weight if the assignment corresponds to $T'$ and negative otherwise). Then +\begin{align*} + w(G_T) &= \sum_{e\in E_T} w(e) = + \alpha \big( (\#Z-12)N + 4 Q_Z\big) + + \beta \big( (\#V-12)N + 4 Q_V\big) \\ + &= \mathrm{const}+ 4(\alpha Q_Z + \beta Q_V). +\end{align*} +So maximizing the weight of such graph encoding would be equivalent to maximizing a linear combination of $Q_Z$ and $Q_V$. + + +\subsection{Algorithm} +We give a high level description of the algorithm to compute an optimal 3-strict assignment. The operations appearing at lines 1,2,4 are respectively described by Algorithms \ref{alg:util},\ref{alg:opt} and \ref{alg:mini}. + + + +\begin{algorithm}[H] + \caption{Optimal 3-strict assignment} + \label{alg:total} + \begin{algorithmic}[1] + \Function{Optimal 3-strict assignment}{$N$, $(c_v)_{v\in V}$, $T'$} + \State $(n_v)_{v\in V} \leftarrow$ \Call{Compute optimal utilization}{$N$, $(c_v)_{v\in V}$} + \State $(T_i)_{1\le i\le N} \leftarrow$ \Call{Compute candidate assignment}{$N$, $(n_v)_{v\in V}$} + \If {there was a previous assignment $T'$} + \State $T \leftarrow$ \Call{Minimization of transfers}{$(T_i)_{1\le i\le N}$, $(T'_i)_{1\le i\le N}$} + \EndIf + \State \Return $T$. + \EndFunction + \end{algorithmic} +\end{algorithm} + +We give some considerations of worst case complexity for these algorithms. In the following, we assume $N>\#V>\#Z$. The complexity of Algorithm \ref{alg:total} is $O(N^3\# Z)$ if we assume \eqref{hyp:A} and $O(N^3 \#Z \#V)$ if we assume \eqref{hyp:B}. + +Algorithm \ref{alg:util} can be implemented with complexity $O(\#V^2)$. The complexity of the function call at line \ref{lin:subutil} is $O(\#V)$. The difference between the sum of the subutilizations and $3N$ is at most the sum of the rounding errors when computing the $\hat{n}_v$. Hence it is bounded by $\#V$ and the loop at line \ref{lin:loopsub} is iterated at most $\#V$ times. Finding the minimizing $v$ at line \ref{lin:findmin} takes $O(\#V)$ operations (naively, we could also use a heap). + +Algorithm \ref{alg:opt} can be implemented with complexity $O(N^3\times \#Z)$. The flow graph has $O(N+\#Z)$ vertices and $O(N\times \#Z)$ edges. Dinic's algorithm has complexity $O(\#\mathrm{Vertices}^2\#\mathrm{Edges})$ hence in our case it is $O(N^3\times \#Z)$. + +Algorithm \ref{alg:mini} can be implented with complexity $O(N^3\# Z)$ under \eqref{hyp:A} and $O(N^3 \#Z \#V)$ under \eqref{hyp:B}. +The graph $G_T$ has $O(N)$ vertices and $O(N\times \#Z)$ edges under assumption \eqref{hyp:A} and respectively $O(N\times \#Z)$ vertices and $O(N\times \#V)$ edges under assumption \eqref{hyp:B}. The loop at line \ref{lin:repeat} is iterated at most $N$ times since the distance between $T$ and $T'$ decreases at every iteration. Bellman-Ford algorithm has complexity $O(\#\mathrm{Vertices}\#\mathrm{Edges})$, which in our case amounts to $O(N^2\# Z)$ under \eqref{hyp:A} and $O(N^2 \#Z \#V)$ under \eqref{hyp:B}. + +\begin{algorithm} + \caption{Computation of the optimal utilization} + \label{alg:util} + \begin{algorithmic}[1] +\Function{Compute optimal utilization}{$N$, $(c_v)_{v\in V}$} + \State $(\hat{n}_v)_{v\in V} \leftarrow $ \Call{Compute subutilization}{$N$, $(c_v)_{v\in V}$} \label{lin:subutil} + \While{$\sum_{v\in V} \hat{n}_v < 3N$} \label{lin:loopsub} + \State Pick $v\in V$ minimizing $\frac{c_v}{\hat{n}_v+1}$ and such that + $\sum_{v'\in z_v} \hat{n}_{v'} < N$ \label{lin:findmin} + \State $\hat{n}_v \leftarrow \hat{n}_v+1$ + \EndWhile + \State \Return $(\hat{n}_v)_{v\in V}$ +\EndFunction +\State + +\Function{Compute subutilization}{$N$, $(c_v)_{v\in V}$} + \State $R \leftarrow 3$ +\For{$v\in V$} +\State $\hat{n}_v \leftarrow \mathrm{unset}$ +\EndFor +\For{$z\in Z$} +\State $c_z \leftarrow \sum_{v\in z} c_v$ +\EndFor +\State $C \leftarrow \sum_{z\in Z} c_z$ +\While{$\exists z \in Z$ such that $R\times c_{z} > C$} +\For{$v\in z$} +\State $\hat{n}_v \leftarrow \left\lfloor \frac{c_v}{c_z} N \right\rfloor$ +\EndFor +\State $C \leftarrow C-c_z$ +\State $R\leftarrow R-1$ +\EndWhile +\For{$v\in V$} +\If{$\hat{n}_v = \mathrm{unset}$} +\State $\hat{n}_v \leftarrow \left\lfloor \frac{Rc_v}{C} N \right\rfloor$ +\EndIf +\EndFor +\State \Return $(\hat{n}_v)_{v\in V}$ +\EndFunction + \end{algorithmic} +\end{algorithm} + +\begin{algorithm} + \caption{Computation of a candidate assignment} + \label{alg:opt} + \begin{algorithmic}[1] + \Function{Compute candidate assignment}{$N$, $(n_v)_{v\in V}$} + \State Compute the flow graph $G$ + \State Compute the maximal flow $f$ using Dinic's algorithm with randomized neighbours enumeration + \State Construct the assignment $(T_i)_{1\le i\le N}$ from $f$ + \State \Return $(T_i)_{1\le i\le N}$ + \EndFunction + \end{algorithmic} +\end{algorithm} + + +\begin{algorithm} + \caption{Minimization of the number of transfers} + \label{alg:mini} + \begin{algorithmic}[1] + \Function{Minimization of transfers}{$(T_i)_{1\le i\le N}$, $(T'_i)_{1\le i\le N}$} + \State Construct the graph encoding $G_T$ + \Repeat \label{lin:repeat} + \State Find a negative cycle $\gamma$ using Bellman-Ford algorithm on $G_T$ + \State Reverse the orientations and weights of edges in $\gamma$ + \Until{no negative cycle is found} + \State Update $(T_i)_{1\le i\le N}$ from $G_T$ + \State \Return $(T_i)_{1\le i\le N}$ + \EndFunction + \end{algorithmic} +\end{algorithm} + +\newpage + +\section{Computation of a 3-non-strict assignment} + +\subsection{Choices of optimality} + +In this mode, we primarily want to store every partition on three nodes, and only secondarily try to spread the nodes among different zone. So we make the choice of not taking the zone repartition in the criterion of optimality. + +We try to maximize $s^*$ defined in \eqref{eq:optimal}. So we can compute the optimal utilizations $(n_v)_{v\in V}$ with the only constraint that $n_v \le N$ for every node $v$. As in the previous section, we start with a sub-utilization proportional to $c_v$ (and capped at $N$), and we iteratively increase the $\hat{n}_v$ that is less than $N$ and maximizes the quantity $c_v/(\hat{n}_v+1)$, until the total sum is $3N$. + +\subsection{Computation of a candidate assignment} + +To compute a candidate assignment (that does not optimize zone spreading nor distance to a previous assignment yet), we can use the folowing flow problem. + +Define the oriented weighted graph $(X,E)$. The set of vertices $X$ contains the source $\mathbf{s}$, the sink $\mathbf{t}$, vertices +$\mathbf{x}_p, \mathbf{u}^+_p, \mathbf{u}^-_p$ for every partition $p$, vertices $\mathbf{y}_{p,z}$ for every partition $p$ and zone $z$, and vertices $\mathbf{z}_v$ for every node $v$. + +The set of edges is composed of the following arcs: +\begin{itemize} + \item ($\mathbf{s}$,$\mathbf{x}_p$, 3) for every partition $p$; + \item ($\mathbf{x}_p$,$\mathbf{u}^+_p$, 3) for every partition $p$; + \item ($\mathbf{x}_p$,$\mathbf{u}^-_p$, 2) for every partition $p$; + \item ($\mathbf{u}^+_p$,$\mathbf{y}_{p,z}$, 1) for every partition $p$ and zone $z$; + \item ($\mathbf{u}^-_p$,$\mathbf{y}_{p,z}$, 2) for every partition $p$ and zone $z$; + \item ($\mathbf{y}_{p,z}$,$\mathbf{z}_v$, 1) for every partition $p$, zone $z$ and node $v\in z$; + \item ($\mathbf{z}_v$, $\mathbf{t}$, $n_v$) for every node $v$; +\end{itemize} + +One can check that any maximal flow in this graph corresponds to an assignment of partitions to nodes. In such a flow, all the arcs from $\mathbf{s}$ and to $\mathbf{t}$ are saturated. The arc from $\mathbf{y}_{p,z}$ to $\mathbf{z}_v$ is saturated if and only if $p$ is associated to~$v$. +Finally the flow from $\mathbf{x}_p$ to $\mathbf{y}_{p,z}$ can go either through $\mathbf{u}^+_p$ or $\mathbf{u}^-_p$. + + + +\subsection{Maximal spread and minimal transfers} +Notice that if the arc $\mathbf{u}_p^+\mathbf{y}_{p,z}$ is not saturated but there is some flow in $\mathbf{u}_p^-\mathbf{y}_{p,z}$, then it is possible to transfer a unit of flow from the path $\mathbf{x}_p\mathbf{u}_p^-\mathbf{y}_{p,z}$ to the path $\mathbf{x}_p\mathbf{u}_p^+\mathbf{y}_{p,z}$. So we can always find an equivalent maximal flow $f^*$ that uses the path through $\mathbf{u}_p^-$ only if the path through $\mathbf{u}_p^+$ is saturated. + +We will use this fact to consider the amount of flow going through the vertices $\mathbf{u}^+$ as a measure of how well the partitions are spread over nodes belonging to different zones. If the partition $p$ is associated to 3 different zones, then a flow of 3 will cross $\mathbf{u}_p^+$ in $f^*$ (i.e. a flow of 0 will cross $\mathbf{u}_p^+$). If $p$ is associated to two zones, a flow of $2$ will cross $\mathbf{u}_p^+$. If $p$ is associated to a single zone, a flow of $1$ will cross $\mathbf{u}_p^+$. + +Let $N_1, N_2, N_3$ be the number of partitions associated to respectively 1,2 and 3 distinct zones. We will optimize a linear combination of these variables using the discovery of positively weighted circuits in a graph. + +At the same step, we will also optimize the distance to a previous assignment $T'$. Let $\alpha> \beta> \gamma \ge 0$ be three parameters. + +Given the flow $f$, let $G_f=(X',E_f)$ be the multi-graph where $X' = X\setminus\{\mathbf{s},\mathbf{t}\}$. The set $E_f$ is composed of the arcs: +\begin{itemize} +\item As many arcs from $(\mathbf{x}_p, \mathbf{u}^+_p,\alpha), (\mathbf{x}_p, \mathbf{u}^+_p,\beta), (\mathbf{x}_p, \mathbf{u}^+_p,\gamma)$ (selected in this order) as there is flow crossing $\mathbf{u}^+_p$ in $f$; +\item As many arcs from $(\mathbf{u}^+_p, \mathbf{x}_p,-\gamma), (\mathbf{u}^+_p, \mathbf{x}_p,-\beta), (\mathbf{u}^+_p, \mathbf{x}_p,-\alpha)$ (selected in this order) as there is flow crossing $\mathbf{u}^-_p$ in $f$; +\item As many copies of $(\mathbf{x}_p, \mathbf{u}^-_p,0)$ as there is flow through $\mathbf{u}^-_p$; +\item As many copies of $(\mathbf{u}^-_p,\mathbf{x}_p,0)$ so that the number of arcs between these two vertices is 2; +\item $(\mathbf{u}^+_p,\mathbf{y}_{p,z}, 0)$ if the flow between these vertices is 1, and the opposite arc otherwise; +\item as many copies of $(\mathbf{u}^-_p,\mathbf{y}_{p,z}, 0)$ as the flow between these vertices, and as many copies of the opposite arc as 2~$-$~the flow; +\item $(\mathbf{y}_{p,z},\mathbf{z}_v, \pm1)$ if it is saturated in $f$, with $+1$ if $v\in T'_p$ and $-1$ otherwise; +\item $(\mathbf{z}_v,\mathbf{y}_{p,z}, \pm1)$ if it is not saturated in $f$, with $+1$ if $v\notin T'_p$ and $-1$ otherwise. +\end{itemize} +To summarize, arcs are oriented left to right if they correspond to a presence of flow in $f$, and right to left if they correspond to an absence of flow. They are positively weighted if we want them to stay at their current state, and negatively if we want them to switch. Let us compute the weight of such graph. + +\begin{multline*} + w(G_f) = \sum_{e\in E_f} w(e_f) \\ + = + (\alpha - \beta -\gamma) N_1 + (\alpha +\beta - \gamma) N_2 + (\alpha+\beta+\gamma) N_3 + \\ + + \#V\times N - 4 \sum_p 3-\#(T_p\cap T'_p) \\ + =(\#V-12+\alpha-\beta-\gamma)\times N + 4Q_V + 2\beta N_2 + 2(\beta+\gamma) N_3 \\ +\end{multline*} + +As for the mode 3-strict, one can check that the difference of two such graphs corresponding to the same $(n_v)$ is always eulerian. Hence we can navigate in this class with the same greedy algorithm that discovers positive cycles and flips them. + +The function that we optimize is +$$ +2Q_V + \beta N_2 + (\beta+\gamma) N_3. +$$ +The choice of parameters $\beta$ and $\gamma$ should be lead by the following question: For $\beta$, where to put the tradeoff between zone dispersion and distance to the previous configuration? For $\gamma$, do we prefer to have more partitions spread between 2 zones, or have less between at least 2 zones but more between 3 zones. + +The quantity $Q_V$ varies between $0$ and $3N$, it should be of order $N$. The quantity $N_2+N_3$ should also be of order $N$ (it is exactly $N$ in the strict mode). So the two terms of the function are comparable. + + +\bibliography{optimal_layout} +\bibliographystyle{ieeetr} + +\end{document} + + + diff --git a/doc/sticker/Garage.pdf b/doc/sticker/Garage.pdf new file mode 100644 index 00000000..1bfb4a16 Binary files /dev/null and b/doc/sticker/Garage.pdf differ diff --git a/doc/sticker/Garage.png b/doc/sticker/Garage.png new file mode 100644 index 00000000..1bfd61b1 Binary files /dev/null and b/doc/sticker/Garage.png differ diff --git a/doc/sticker/Garage_NGI.pdf b/doc/sticker/Garage_NGI.pdf index 1359a30d..43f511e7 100644 Binary files a/doc/sticker/Garage_NGI.pdf and b/doc/sticker/Garage_NGI.pdf differ diff --git a/doc/talks/.gitignore b/doc/talks/.gitignore new file mode 100644 index 00000000..9b421064 --- /dev/null +++ b/doc/talks/.gitignore @@ -0,0 +1 @@ +.direnv/ diff --git a/doc/talks/2023-09-20-ocp/.gitignore b/doc/talks/2023-09-20-ocp/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2023-09-20-ocp/Makefile b/doc/talks/2023-09-20-ocp/Makefile new file mode 100644 index 00000000..554f7b97 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/Makefile @@ -0,0 +1,34 @@ +ASSETS=assets/consistent_hashing_1.pdf \ + assets/consistent_hashing_2.pdf \ + assets/consistent_hashing_3.pdf \ + assets/consistent_hashing_4.pdf \ + assets/garage_tables.pdf \ + assets/consensus.pdf_tex \ + assets/lattice1.pdf_tex \ + assets/lattice2.pdf_tex \ + assets/lattice3.pdf_tex \ + assets/lattice4.pdf_tex \ + assets/lattice5.pdf_tex \ + assets/lattice6.pdf_tex \ + assets/lattice7.pdf_tex \ + assets/lattice8.pdf_tex \ + assets/latticeB_1.pdf_tex \ + assets/latticeB_2.pdf_tex \ + assets/latticeB_3.pdf_tex \ + assets/latticeB_4.pdf_tex \ + assets/latticeB_5.pdf_tex \ + assets/latticeB_6.pdf_tex \ + assets/latticeB_7.pdf_tex \ + assets/latticeB_8.pdf_tex \ + assets/latticeB_9.pdf_tex \ + assets/latticeB_10.pdf_tex \ + assets/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +assets/%.pdf: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +assets/%.pdf_tex: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2023-09-20-ocp/abstract.md b/doc/talks/2023-09-20-ocp/abstract.md new file mode 100644 index 00000000..b2658868 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/abstract.md @@ -0,0 +1,39 @@ +### (fr) Garage, un système de stockage de données géo-distribué léger et robuste + +Garage est un système de stockage de données léger, géo-distribué, qui +implémente le protocole de stockage S3 de Amazon. Garage est destiné +principalement à l'auto-hébergement sur du matériel courant d'occasion. À ce +titre, il doit tolérer un grand nombre de pannes: coupures de courant, coupures +de connexion Internet, pannes de machines, ... Il doit également être facile à +déployer et à maintenir, afin de pouvoir être facilement utilisé par des +amateurs ou des petites organisations. + +Cette présentation vous proposera un aperçu de Garage et du choix technique +principal qui rend un système comme Garage possible: le refus d'utiliser des +algorithmes de consensus, remplacés avantageusement par des méthodes à +cohérence faible. Notre modèle est fortement inspiré de la base de donnée +Dynamo (DeCandia et al, 2007), et fait usage des types de données CRDT (Shapiro +et al, 2011). Nous exploreront comment ces méthodes s'appliquent à la +construction de l'abstraction "stockage objet" dans un système distribué, et +quelles autres abstractions peuvent ou ne peuvent pas être construites dans ce +modèle. + +### (en) Garage, a lightweight and robust geo-distributed data storage system + +Garage is a lightweight geo-distributed data store that implements the Amazon +S3 object storage protocol. Garage is meant primarily for self-hosting at home +on second-hand commodity hardware, meaning it has to tolerate a wide variety of +failure scenarios such as power cuts, Internet disconnections and machine +crashes. It also has to be easy to deploy and maintain, so that hobbyists and +small organizations can use it without trouble. + +This talk will present Garage and the key technical choice that made Garage +possible: refusing to use consensus algorithms and using instead weak +consistency methods, with a model that is loosely based on that of the Dynamo +database (DeCandia et al, 2007) and that makes heavy use of conflict-free +replicated data types (Shapiro et al, 2011). We will explore how these methods +are suited to building the "object store" abstraction in a distributed system, +and what other abstractions are possible or impossible to build in this model. + + + diff --git a/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png b/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png new file mode 100644 index 00000000..445284a3 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/AGPLv3_Logo.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme.png b/doc/talks/2023-09-20-ocp/assets/aerogramme.png new file mode 100644 index 00000000..3aabe3ad Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme.svg b/doc/talks/2023-09-20-ocp/assets/aerogramme.svg new file mode 100644 index 00000000..0c1ee127 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/aerogramme.svg @@ -0,0 +1,1241 @@ + + + + + + K2V APIS3 APIAerogramme + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +IMAPIMAPIMAPMessageindexMessagebodies diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf new file mode 100644 index 00000000..71a90f26 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png new file mode 100644 index 00000000..fb81b460 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf new file mode 100644 index 00000000..87e42eed Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png new file mode 100644 index 00000000..f9e2df14 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_components2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf new file mode 100644 index 00000000..0606e059 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png new file mode 100644 index 00000000..c3b015a1 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_datatype.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf new file mode 100644 index 00000000..8fea81c7 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png new file mode 100644 index 00000000..ed2077d9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/aerogramme_keys.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/alex.jpg b/doc/talks/2023-09-20-ocp/assets/alex.jpg new file mode 100644 index 00000000..eac0f0a9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/alex.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/atuin.jpg b/doc/talks/2023-09-20-ocp/assets/atuin.jpg new file mode 100644 index 00000000..f2fbd61d Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/atuin.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/compatibility.png b/doc/talks/2023-09-20-ocp/assets/compatibility.png new file mode 100644 index 00000000..ce364a9b Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/compatibility.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/consensus.svg b/doc/talks/2023-09-20-ocp/assets/consensus.svg new file mode 100644 index 00000000..8321e383 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consensus.svg @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + $\bot$ + + + + $x$ + + + $propose(x) / x$ + $propose(y) / x$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg new file mode 100644 index 00000000..f8d24fd8 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_1.svg @@ -0,0 +1,301 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg new file mode 100644 index 00000000..5ac8faf6 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_2.svg @@ -0,0 +1,334 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg new file mode 100644 index 00000000..fdfd3efc --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_3.svg @@ -0,0 +1,358 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg new file mode 100644 index 00000000..95ed0e02 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/consistent_hashing_4.svg @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg b/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg new file mode 100644 index 00000000..c298c22b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/deuxfleurs.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + D + F + diff --git a/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png b/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png new file mode 100644 index 00000000..7c7411cd Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/endpoint-latency-dc.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf new file mode 100644 index 00000000..a54a163c Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage.drawio.png new file mode 100644 index 00000000..386dd862 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png new file mode 100644 index 00000000..8562fbcf Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..422c9343 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2a.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..05a9710e Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage2b.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf new file mode 100644 index 00000000..a94b3572 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.pdf differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png new file mode 100644 index 00000000..2e7b5af0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/garage_sync.drawio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/garage_tables.svg b/doc/talks/2023-09-20-ocp/assets/garage_tables.svg new file mode 100644 index 00000000..c7172713 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/garage_tables.svg @@ -0,0 +1,537 @@ + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + Object + + + + + + bucket + + + + + + file path + + = partition key + + = sort key + + + + + + Version 1 + deleted + + + + + + + Version 2 + id + + size + MIME type + ... + + + + + + Version + + id + h(block 1) + h(block 2) + ... + + + + + Data block + + hash + data + + + + Objects table + Versions table + Blocks table + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice1.svg b/doc/talks/2023-09-20-ocp/assets/lattice1.svg new file mode 100644 index 00000000..8bfa5aa7 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice1.svg @@ -0,0 +1,433 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice2.svg b/doc/talks/2023-09-20-ocp/assets/lattice2.svg new file mode 100644 index 00000000..adcd92cb --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice2.svg @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice3.svg b/doc/talks/2023-09-20-ocp/assets/lattice3.svg new file mode 100644 index 00000000..640dc468 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice3.svg @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice4.svg b/doc/talks/2023-09-20-ocp/assets/lattice4.svg new file mode 100644 index 00000000..b2a99e28 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice4.svg @@ -0,0 +1,525 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice5.svg b/doc/talks/2023-09-20-ocp/assets/lattice5.svg new file mode 100644 index 00000000..bc6b7195 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice5.svg @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice6.svg b/doc/talks/2023-09-20-ocp/assets/lattice6.svg new file mode 100644 index 00000000..176b1715 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice6.svg @@ -0,0 +1,553 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice7.svg b/doc/talks/2023-09-20-ocp/assets/lattice7.svg new file mode 100644 index 00000000..7ce8bda8 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice7.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/lattice8.svg b/doc/talks/2023-09-20-ocp/assets/lattice8.svg new file mode 100644 index 00000000..c94a69b2 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/lattice8.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg new file mode 100644 index 00000000..92232a1b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_1.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg new file mode 100644 index 00000000..34c24e0d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_10.svg @@ -0,0 +1,715 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + return $\{b\}$ + + $\to \{b\}$ + ${\Large\textbf{??!}}$~~~~~$\{a\} \not\sqsubseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg new file mode 100644 index 00000000..c07cba2b --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_2.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + $\not\sqsupseteq \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg new file mode 100644 index 00000000..198d1f5d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_3.svg @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg new file mode 100644 index 00000000..c5f6148d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_4.svg @@ -0,0 +1,587 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg new file mode 100644 index 00000000..c2b668be --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_5.svg @@ -0,0 +1,604 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg new file mode 100644 index 00000000..980823fc --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_6.svg @@ -0,0 +1,632 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg new file mode 100644 index 00000000..154c0b7d --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_7.svg @@ -0,0 +1,654 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + $read()$: + ; + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg new file mode 100644 index 00000000..21766415 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_8.svg @@ -0,0 +1,671 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + + diff --git a/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg b/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg new file mode 100644 index 00000000..b60f8afe --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/latticeB_9.svg @@ -0,0 +1,699 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + + + $write(\{b\})$: + $read()$: + $\not\sqsupseteq \{b\}$ + $\sqsupseteq \{b\} \to$ OK + $\not\sqsupseteq \{b\}$ + + + $\to \{a\}$ + return $\{a\}$ + + $\to \{\}$ + + $\to \{\}$ + $read()$: + ; + return $\{b\}$ + + $\to \{b\}$ + + diff --git a/doc/talks/2023-09-20-ocp/assets/location-aware.png b/doc/talks/2023-09-20-ocp/assets/location-aware.png new file mode 100644 index 00000000..f5966865 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/location-aware.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/logo_chatons.png b/doc/talks/2023-09-20-ocp/assets/logo_chatons.png new file mode 100644 index 00000000..890cf17e Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/logo_chatons.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/map.png b/doc/talks/2023-09-20-ocp/assets/map.png new file mode 100644 index 00000000..1dff3ab6 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/map.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/minio.png b/doc/talks/2023-09-20-ocp/assets/minio.png new file mode 100644 index 00000000..a71e9ccc Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/minio.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/neptune.jpg b/doc/talks/2023-09-20-ocp/assets/neptune.jpg new file mode 100644 index 00000000..61fcbff6 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/neptune.jpg differ diff --git a/doc/talks/2023-09-20-ocp/assets/rust_logo.png b/doc/talks/2023-09-20-ocp/assets/rust_logo.png new file mode 100644 index 00000000..0e4809ec Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/rust_logo.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png new file mode 100644 index 00000000..c92f0774 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_stats.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png new file mode 100644 index 00000000..849b8336 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_status.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png new file mode 100644 index 00000000..86dcce89 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_status_broken.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png new file mode 100644 index 00000000..e7d4e288 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_get.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png new file mode 100644 index 00000000..472312a0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/scr_garage_worker_list.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide1.png b/doc/talks/2023-09-20-ocp/assets/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide2.png b/doc/talks/2023-09-20-ocp/assets/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slide3.png b/doc/talks/2023-09-20-ocp/assets/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slide3.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB1.png b/doc/talks/2023-09-20-ocp/assets/slideB1.png new file mode 100644 index 00000000..b14b6070 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB1.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB2.png b/doc/talks/2023-09-20-ocp/assets/slideB2.png new file mode 100644 index 00000000..a881a796 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB2.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slideB3.png b/doc/talks/2023-09-20-ocp/assets/slideB3.png new file mode 100644 index 00000000..830709d2 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/assets/slideB3.png differ diff --git a/doc/talks/2023-09-20-ocp/assets/slides.svg b/doc/talks/2023-09-20-ocp/assets/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/2023-09-20-ocp/assets/slidesB.svg b/doc/talks/2023-09-20-ocp/assets/slidesB.svg new file mode 100644 index 00000000..c0a6e97c --- /dev/null +++ b/doc/talks/2023-09-20-ocp/assets/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database diff --git a/doc/talks/2023-09-20-ocp/talk.pdf b/doc/talks/2023-09-20-ocp/talk.pdf new file mode 100644 index 00000000..f7b96ef9 Binary files /dev/null and b/doc/talks/2023-09-20-ocp/talk.pdf differ diff --git a/doc/talks/2023-09-20-ocp/talk.tex b/doc/talks/2023-09-20-ocp/talk.tex new file mode 100644 index 00000000..21ce54b0 --- /dev/null +++ b/doc/talks/2023-09-20-ocp/talk.tex @@ -0,0 +1,1008 @@ +\nonstopmode +\documentclass[aspectratio=169]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\subtitle{a lightweight and robust geo-distributed data storage system} +\author{Alex Auvolat, Deuxfleurs} +\date{OCamlPro, 2023-09-20} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{assets/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{assets/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize (we want good uptime/availability with low supervision)} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make a \underline{stable} system} + + Enterprise-grade systems typically employ: + \vspace{1em} + \begin{itemize} + \item RAID + \item Redundant power grid + UPS + \item Redundant Internet connections + \item Low-latency links + \item ... + \end{itemize} + \vspace{1em} + $\to$ it's costly and only worth it at DC scale +\end{frame} + +\begin{frame} + \frametitle{How to make a \underline{resilient} system} + + \only<1,4-5>{ + Instead, we use: + \vspace{1em} + \begin{itemize} + \item \textcolor<2->{gray}{Commodity hardware (e.g. old desktop PCs)} + \vspace{.5em} + \item<4-> \textcolor<5->{gray}{Commodity Internet (e.g. FTTB, FTTH) and power grid} + \vspace{.5em} + \item<5-> \textcolor<6->{gray}{\textbf{Geographical redundancy} (multi-site replication)} + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/neptune.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/atuin.jpg} + \end{center} + } + \only<6>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{assets/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{assets/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec (e.g.~locks) + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[height=6em]{../2020-12-02_wide-team/img/Amazon-S3.jpg} + \hspace{3em} + \includegraphics[height=5em]{assets/minio.png} + \hspace{3em} + \includegraphics[height=6em]{../../logo/garage_hires_crop.png} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + + MinIO is self-hostable but not suited for geo-distributed deployments + + \vspace{1em} + + \textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store} +\end{frame} + + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a key-value store: + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|p{8cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + +\end{frame} + + +\begin{frame} + \frametitle{Two big problems} + \begin{enumerate} + \item \textbf{How to place data on different nodes?}\\ + \vspace{1em} + \underline{Constraints:} heterogeneous hardware\\ + \underline{Objective:} $n$ copies of everything, maximize usable capacity, maximize resilience\\ + \vspace{1em} + $\to$ the Dynamo model + optimization algorithms + \vspace{2em} + \item<2-> \textbf{How to guarantee consistency?}\\ + \vspace{1em} + \underline{Constraints:} slow network (geographical distance), node unavailability/crashes\\ + \underline{Objective:} maximize availability, read-after-write guarantee\\ + \vspace{1em} + $\to$ CRDTs, monotonicity, read and write quorums + \end{enumerate} +\end{frame} + +\section{Problem 1: placing data} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \textbf{Two keys:} + \begin{itemize} + \item Partition key: used to divide data into partitions {\small (a.k.a.~shards)} + \item Sort key: used to identify items inside a partition + \end{itemize} + + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|l|p{3cm}|} + \hline + \textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & \texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \begin{itemize} + \item Data with different partition keys is stored independently,\\ + on a different set of nodes\\ + \vspace{.5em} + $\to$ no easy way to list all partition keys\\ + $\to$ no cross-shard transactions\\ + \vspace{2em} + \item Placing data: hash the partition key, select nodes accordingly\\ + \vspace{.5em} + $\to$ distributed hash table (DHT) + \vspace{2em} + \item For a given value of the partition key, items can be listed using their sort keys + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How to spread files over different cluster nodes?} + \textbf{Consistent hashing (Dynamo):} + \vspace{1em} + + \begin{center} + \only<1>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_1.pdf}}% + \only<2>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_2.pdf}}% + \only<3>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_3.pdf}}% + \only<4>{\includegraphics[width=.40\columnwidth]{assets/consistent_hashing_4.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Constraint: location-awareness} + \begin{center} + \includegraphics[width=\linewidth]{assets/location-aware.png} + \end{center} + \vspace{2em} + Garage replicates data on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{Constraint: location-awareness} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/map.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Issues with consistent hashing} + \begin{itemize} + \item Consistent hashing doesn't dispatch data based on geographical location of nodes + \vspace{1em} + \item<2-> Geographically aware adaptation, try 1:\\ + data quantities not well balanced between nodes + \vspace{1em} + \item<3-> Geographically aware adaptation, try 2:\\ + too many reshuffles when adding/removing nodes + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How to spread files over different cluster nodes?} + \textbf{Garage's method: build an index table} + \vspace{1em} + + Realization: we can actually precompute an optimal solution + \vspace{1em} + + \visible<2->{ + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & Io (jupiter) & Drosera (atuin) & Courgette (neptune) \\ + \hline + Partition 1 & Datura (atuin) & Courgette (neptune) & Io (jupiter) \\ + \hline + Partition 2 & Io(jupiter) & Celeri (neptune) & Drosera (atuin) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & Io (jupiter) & Drosera (atuin) \\ + \hline + \end{tabular} + \end{center} + } + \vspace{1em} + \visible<3->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } +\end{frame} + +\begin{frame} + \frametitle{The relationship between \emph{partition} and \emph{partition key}} + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} + \vspace{1em} + \textbf{To read or write an item:} hash partition key + \\ \hspace{5cm} $\to$ determine partition number (first 8 bits) + \\ \hspace{5cm} $\to$ find associated nodes +\end{frame} + +\begin{frame} + \frametitle{Garage's internal data structures} + \centering + \includegraphics[width=.75\columnwidth]{assets/garage_tables.pdf} +\end{frame} + +\begin{frame} + \frametitle{Storing and retrieving files} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{assets/garage2a.drawio.pdf}}% + \only<2>{\includegraphics[width=.45\linewidth]{assets/garage2b.drawio.pdf}}% + \end{center} +\end{frame} + +\section{Problem 2: ensuring consistency} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Consensus-based systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Leader-based:} a leader is elected to coordinate + all reads and writes + \vspace{1em} + \item \textbf{Linearizability} of all operations\\ + (strongest consistency guarantee) + \vspace{1em} + \item Any sequential specification can be implemented as a \textbf{replicated state machine} + \vspace{1em} + \item \textbf{Costly}, the leader is a bottleneck; + leader elections on failure take time + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}{7cm} \visible<2->{ + \textbf{Weakly consistent systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Nodes are equivalent}, any node + can originate a read or write operation + \vspace{1em} + \item \textbf{Read-after-write consistency} with quorums, + eventual consistency without + \vspace{1em} + \item \textbf{Operations have to commute}, i.e.~we + can only implement CRDTs + \vspace{1em} + \item \textbf{Fast}, no single bottleneck;\\ + works the same with offline nodes + \end{itemize} + } \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + \begin{center} + \textbf{From a theoretical point of view:}\\ + + \end{center} + \vspace{2em} + + \hspace{1em} + \begin{minipage}{6.5cm} + \underline{Consensus-based systems:} + + \vspace{1em} + + Require \textbf{additional assumptions} such as a fault detector or a strong RNG\\ + (FLP impossibility theorem) + \end{minipage} + \hfill + \begin{minipage}{6.5cm} + \underline{Weakly consistent systems:} + + \vspace{1em} + + Can be implemented in \textbf{any\\asynchronous message passing\\distributed system} with node crashes + \end{minipage} + \hspace{1em} + + \vspace{3em} + \begin{center} + They represent \textbf{different classes of computational capability}\\ + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Consensus vs weak consistency} + \begin{center} + \textbf{The same objects cannot be implemented in both models.} + \end{center} + \vspace{2em} + + \hspace{1em} + \begin{minipage}{6.5cm} + \underline{Consensus-based systems:} + + \vspace{1em} + + \textbf{Any sequential specification}\\~ + + \vspace{1em} + \textbf{Easier to program for}: just write your program as if it were sequential on a single machine + + \end{minipage} + \hfill + \begin{minipage}{6.5cm} + \underline{Weakly consistent systems:} + + \vspace{1em} + + \textbf{Only CRDTs}\\(conflict-free replicated data types) + + \vspace{1em} + Part of the complexity is \textbf{reported to the consumer of the API}\\~ + \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Understanding the power of consensus} + \textbf{Consensus:} an API with a single operation, $propose(x)$ + \begin{enumerate} + \item nodes all call $propose(x)$ with their proposed value; + \item nodes all receive the same value as a return value, which is one of the proposed values + \end{enumerate} + \vspace{1em} + + \visible<2->{ + \textbf{Equivalent to} a distributed algorithm that gives a total order on all requests + } + \vspace{1em} + + \visible<3->{ + \textbf{Implemented by} this simple replicated state machine: + \vspace{.5em} + \begin{figure} + \centering + \def\svgwidth{.5\textwidth} + \large + \import{assets/}{consensus.pdf_tex} + \end{figure} + \vspace{1em} + } +\end{frame} + +\begin{frame} + \frametitle{Can my object be implemented without consensus?} + \underline{Given the specification of an API:} + \vspace{2em} + \begin{itemize} + \item \textbf{Using this API, we can implement the consensus object} (the $propose$ function)\\ + $\to$ the API is equivalent to consensus/total ordering of messages\\ + $\to$ the API cannot be implemented in a weakly consistent system + \vspace{2em} + \item<2-> \textbf{This API can be implemented using only weak primitives}\\ + (e.g. in the asynchronous message passing model with no further assumption)\\ + $\to$ the API is strictly weaker than consensus\\ + $\to$ we can implement it in Garage! + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Why avoid consensus?} + Consensus can be implemented reasonably well in practice, so why avoid it? + \vspace{2em} + \begin{itemize} + \item \textbf{Software complexity:} RAFT and PAXOS are complex beasts;\\ + harder to prove, harder to reason about + \vspace{1.5em} + \item \textbf{Performance issues:} + \vspace{1em} + \begin{itemize} + \item Theoretical requirements (RNG, failure detector) translate into \textbf{practical costs} + \vspace{1em} + \item The leader is a \textbf{bottleneck} for all requests;\\ + even in leaderless approaches, \textbf{all nodes must process all operations in order} + \vspace{1em} + \item Particularly \textbf{sensitive to higher latency} between nodes + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/endpoint-latency-dc.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{What can we implement without consensus?} + \begin{itemize} + \item Any \textbf{conflict-free replicated data type} (CRDT) + \vspace{1em} + \item<2-> Non-transactional key-value stores such as S3 are equivalent to a simple CRDT:\\ + a map of \textbf{last-writer-wins registers} (each key is its own CRDT) + \vspace{1em} + \item<3-> \textbf{Read-after-write consistency} can be implemented + using quorums on read and write operations + \vspace{1em} + \item<4-> \textbf{Monotonicity of reads} can be implemented with repair-on-read\\ + (makes reads more costly, not implemented in Garage) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: read-after-write consistency} + \begin{figure} + \centering + \def\svgwidth{.8\textwidth} + \only<1>{\import{assets/}{lattice1.pdf_tex}}% + \only<2>{\import{assets/}{lattice2.pdf_tex}}% + \only<3>{\import{assets/}{lattice3.pdf_tex}}% + \only<4>{\import{assets/}{lattice4.pdf_tex}}% + \only<5>{\import{assets/}{lattice5.pdf_tex}}% + \only<6>{\import{assets/}{lattice6.pdf_tex}}% + \only<7>{\import{assets/}{lattice7.pdf_tex}}% + \only<8>{\import{assets/}{lattice8.pdf_tex}}% + \end{figure} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: read-after-write consistency} + \textbf{Property:} If node $A$ did an operation $write(x)$ and received an OK response,\\ + \hspace{2cm} and node $B$ starts an operation $read()$ after $A$ received OK,\\ + \hspace{2cm} then $B$ will read a value $x' \sqsupseteq x$. + + \vspace{1em} + + \hspace{1em} + \begin{minipage}{6.8cm} + \textbf{Algorithm $write(x)$:} + \begin{enumerate} + \item Broadcast $write(x)$ to all nodes + \item Wait for $k > n/2$ nodes to reply OK + \item Return OK + \end{enumerate} + \end{minipage} + \hfill + \begin{minipage}{6.8cm} + \vspace{1em} + \textbf{Algorithm $read()$:} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply\\ + with values $x_1, \dots, x_k$ + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + \end{minipage} + \hspace{1em} + + \vspace{2em} + \textbf{Why does it work?} There is at least one node at the intersection between the two sets of nodes that replied to each request, that ``saw'' $x$ before the $read()$ started ($x_i \sqsupseteq x$). +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: monotonic-reads consistency} + \begin{figure} + \centering + \def\svgwidth{.8\textwidth} + \only<1>{\import{assets/}{latticeB_1.pdf_tex}}% + \only<2>{\import{assets/}{latticeB_2.pdf_tex}}% + \only<3>{\import{assets/}{latticeB_3.pdf_tex}}% + \only<4>{\import{assets/}{latticeB_4.pdf_tex}}% + \only<5>{\import{assets/}{latticeB_5.pdf_tex}}% + \only<6>{\import{assets/}{latticeB_6.pdf_tex}}% + \only<7>{\import{assets/}{latticeB_7.pdf_tex}}% + \only<8>{\import{assets/}{latticeB_8.pdf_tex}}% + \only<9>{\import{assets/}{latticeB_9.pdf_tex}}% + \only<10>{\import{assets/}{latticeB_10.pdf_tex}}% + \end{figure} +\end{frame} + +\begin{frame} + \frametitle{CRDTs and quorums: monotonic-reads consistency} + \textbf{Property:} If node $A$ did an operation $read()$ and received $x$ as a response,\\ + \hspace{2cm} and node $B$ starts an operation $read()$ after $A$ received $x$,\\ + \hspace{2cm} then $B$ will read a value $x' \sqsupseteq x$. + + \vspace{1em} + + \textbf{Algorithm $monotonic\_read()$:} {\small (a.k.a. repair-on-read)} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply with values $x_1, \dots, x_k$ + \item If $x_i \ne x_j$ for some nodes $i$ and $j$,\\ + \hspace{1cm}then call $write(x_1 \sqcup \dots \sqcup x_k)$ and wait for OK from $k' > n/2$ nodes + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + + \vspace{1em} + + This makes reads slower in some cases, and is \textbf{not implemented in Garage}. +\end{frame} + +\begin{frame} + \frametitle{A hard problem: layout changes} + \begin{itemize} + \item We rely on quorums $k > n/2$ within each partition:\\ + $$n=3,~~~~~~~k\ge 2$$ + \item<2-> When rebalancing, the set of nodes responsible for a partition can change:\\ + $$\{n_A, n_B, n_C\} \to \{n_A, n_D, n_E\}$$ + \vspace{.01em} + \item<3-> During the rebalancing, $D$ and $E$ don't yet have the data,\\ + ~~~~~~~~~~~~~~~~~~~and $B$ and $C$ want to get rid of the data to free up space\\ + \vspace{.2em} + $\to$ quorums only within the new set of nodes don't work\\ + $\to$ how to coordinate? \textbf{currently, we don't...} + + \end{itemize} +\end{frame} + +\section{Operating big Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{assets/scr_garage_status.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.85\linewidth]{assets/scr_garage_status_broken.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{assets/garage.drawio.pdf}}% + \only<2>{\includegraphics[width=.6\linewidth]{assets/garage_sync.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/scr_garage_stats.png}} + \only<2>{\includegraphics[width=.6\linewidth]{assets/scr_garage_worker_list.png}} + \only<3>{\includegraphics[width=.6\linewidth]{assets/scr_garage_worker_get.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \item Multi-HDD support not yet released (soon!) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Wait for v0.9 with multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Current deployments: $< 10$ TB, we don't have much experience with more +\end{frame} + +\section{Going further than the S3 API} + +\begin{frame} + \frametitle{Using Garage for everything} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/slideB1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{assets/slideB2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{assets/slideB3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{K2V Design} + \begin{itemize} + \item A new, custom, minimal API\\ + \vspace{.5em} + \begin{itemize} + \item Single-item operations + \item Operations on ranges and batches of items + \item Polling operations to help implement a PubSub pattern + \end{itemize} + \vspace{1em} + \item<2-> Exposes the partitoning mechanism of Garage\\ + K2V = partition key / sort key / value (like Dynamo) + \vspace{1em} + \item<3-> Weakly consistent, CRDT-friendly\\ + $\to$ no support for transactions (not ACID) + \vspace{1em} + \item<4-> Cryptography-friendly: values are binary blobs + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Handling concurrent values} + \textbf{How to handle concurrency?} Example: + \vspace{1em} + \begin{enumerate} + \item Client $A$ reads the initial value of a key, $x_0$ + \vspace{1em} + \item<2-> Client $B$ also reads the initial value $x_0$ of that key + \vspace{1em} + \item<3-> Client $A$ modifies $x_0$, and writes a new value $x_1$ + \vspace{1em} + \item<4-> Client $B$ also modifies $x_0$, and writes a new value $x'_1$,\\ + without having a chance to first read $x_1$\\ + \vspace{1em} + $\to$ what should the final state be? + \end{enumerate} +\end{frame} + +\begin{frame} + \frametitle{Handling concurrent values} + \begin{itemize} + \item If we keep only $x_1$ or $x'_1$, we risk \textbf{loosing application data} + \vspace{1.5em} + \item<2-> Values are opaque binary blobs, \textbf{K2V cannot resolve conflicts} by itself\\ + (e.g. by implementing a CRDT) + \vspace{1.5em} + \item<3-> Solution: \textbf{we keep both!}\\ + $\to$ the value of the key is now $\{x_1, x'_1\}$\\ + $\to$ the client application can decide how to resolve conflicts on the next read + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Keeping track of causality} + How does K2V know that $x_1$ and $x'_1$ are concurrent? + \vspace{1em} + \begin{itemize} + \item $read()$ returns \textbf{a set of values} and an associated \textbf{causality token}\\ + \vspace{1.5em} + \item<2-> When calling $write()$, the client sends \textbf{the causality token from its last read} + \vspace{1.5em} + \item<3-> The causality token represents the set of values \textbf{already seen by the client}\\ + $\to$ those values are the \textbf{causal past} of the write operation\\ + $\to$ K2V can keep concurrent values and overwrite all ones in the causal past + \vspace{1.5em} + \item<4-> Internally, the causality token is \textbf{a vector clock} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Application: an e-mail storage server} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Aerogramme data model} + \begin{center} + \only<1->{\includegraphics[width=.4\linewidth]{assets/aerogramme_datatype.drawio.pdf}}% + \end{center} + \visible<2->{Aerogramme encrypts all stored values for privacy\\ + (Garage server administrators can't read your mail)} +\end{frame} + +\begin{frame} + \frametitle{Different deployment scenarios} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components1.drawio.pdf}}% + \only<2>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components2.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{A new model for building resilient software} + How to build an application using only Garage as a data store: + \vspace{1em} + \begin{enumerate} + \item Design a data model suited to K2V\\ + {\footnotesize (see Cassandra docs on porting SQL data models to Cassandra)} + \vspace{1em} + \begin{itemize} + \item Use CRDTs or other eventually consistent data types (see e.g. Bayou) + \vspace{1em} + \item Store opaque binary blobs to provide End-to-End Encryption\\ + \end{itemize} + \vspace{1em} + \item<2-> Store big blobs (files) using the S3 API + \vspace{1em} + \item<3-> Let Garage manage sharding, replication, failover, etc. + \end{enumerate} +\end{frame} + +\section{Conclusion} + +\begin{frame} + \frametitle{Perspectives} + \begin{itemize} + \item Fix the consistency issue when rebalancing + \vspace{1em} + \item Write about Garage's architecture and properties,\\ + and about our proposed architecture for (E2EE) apps over K2V+S3 + \vspace{1em} + \item Continue developing Garage; finish Aerogramme; build new applications... + \vspace{1em} + \item Anything else? + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{assets/rust_logo.png} + \includegraphics[width=.13\linewidth]{assets/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-01-12-seed/.gitignore b/doc/talks/2024-01-12-seed/.gitignore new file mode 100644 index 00000000..b7e0ced2 --- /dev/null +++ b/doc/talks/2024-01-12-seed/.gitignore @@ -0,0 +1,10 @@ +* + +!*.txt +!*.md +!*.tex + +!talk.pdf +!Makefile +!.gitignore + diff --git a/doc/talks/2024-01-12-seed/Makefile b/doc/talks/2024-01-12-seed/Makefile new file mode 100644 index 00000000..5f0befbe --- /dev/null +++ b/doc/talks/2024-01-12-seed/Makefile @@ -0,0 +1,10 @@ +ASSETS=../assets/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +assets/%.pdf: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +assets/%.pdf_tex: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-01-12-seed/talk.pdf b/doc/talks/2024-01-12-seed/talk.pdf new file mode 100644 index 00000000..85c3bcde Binary files /dev/null and b/doc/talks/2024-01-12-seed/talk.pdf differ diff --git a/doc/talks/2024-01-12-seed/talk.tex b/doc/talks/2024-01-12-seed/talk.tex new file mode 100644 index 00000000..cd6c4681 --- /dev/null +++ b/doc/talks/2024-01-12-seed/talk.tex @@ -0,0 +1,370 @@ +\nonstopmode +\documentclass[aspectratio=169]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\subtitle{a lightweight and robust geo-distributed data storage system} +\author{Alex Auvolat, Deuxfleurs} +\date{SEED webinar, 2024-01-12} + +\begin{document} + +% \begin{frame} +% \centering +% \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} +% \vspace{1em} +% +% {\large\bf Alex Auvolat, Deuxfleurs Association} +% \vspace{1em} +% +% \url{https://garagehq.deuxfleurs.fr/} +% +% %Matrix channel: \texttt{\#garage:deuxfleurs.fr} +% \end{frame} + +\begin{frame} + %\frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + Member of Deuxfleurs, lead developer of Garage + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{.5em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.6\linewidth, valign=t]{../../logo/garage-notext.png} + \end{column} + \begin{column}{.6\textwidth} + \\\textbf{Garage}\\ + A self-hosted alternative to S3 for object storage + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Stable vs Resilient} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Building a "stable" system:} + \vspace{1em} + + Enterprise-grade systems typically employ: + \vspace{1em} + \begin{itemize} + \item RAID + \item Redundant power grid + UPS + \item Redundant Internet connections + \item Low-latency links + \item ... + \end{itemize} + \vspace{1em} + $\to$ costly, only worth at DC scale\\ + $\to$ still risk of DC-level incident... + \end{minipage} + \hfill + \begin{minipage}{7cm} + \textbf{Building a \underline{resilient} system:} + \vspace{1em} + + An alternative, cheaper way: + \vspace{1em} + \begin{itemize} + \item Commodity hardware \\(e.g. old desktop PCs) + \vspace{.5em} + \item Commodity Internet \\(e.g. FTTB, FTTH) and power grid + \vspace{.5em} + \item \textbf{Geographical redundancy} \\(multi-site replication) + \end{itemize} + \vspace{1.5em} + \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Example: our infrastructure at Deuxfleurs} + \only<1>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/atuin.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Object storage: simpler than file systems} + + \begin{minipage}{6cm} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! + \end{minipage} + \hfill + \begin{minipage}{8cm} + \begin{center} + \vspace{2em} + \includegraphics[height=6em]{../2020-12-02_wide-team/img/Amazon-S3.jpg} + \hspace{2em} + \includegraphics[height=5em]{../assets/minio.png} + + \vspace{2em} + \includegraphics[height=6em]{../../logo/garage_hires_crop.png} + \end{center} + \vspace{1em} + \end{minipage} +\end{frame} + + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a key-value store: + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|p{8cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Implementation: consensus vs weak consistency} + + \hspace{1em} + \begin{minipage}{7cm} + \textbf{Consensus-based systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Leader-based:} a leader is elected to coordinate + all reads and writes + \vspace{1em} + \item Allows for \textbf{sequential reasoning}: + program as if running on a single machine + \vspace{1em} + \item Serializability is one of the \\ + \textbf{strongest consistency guarantees} + \vspace{1em} + \item \textbf{Costly}, the leader is a bottleneck; + leader elections on failure take time + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}{7cm} \visible<2->{ + \textbf{Weakly consistent systems:} + \vspace{1em} + \begin{itemize} + \item \textbf{Nodes are equivalent}, any node + can originate a read or write operation + \vspace{1em} + \item \textbf{Operations must be independent}, + conflicts are resolved after the fact + \vspace{1em} + \item Strongest achievable consistency:\\ + \textbf{read-after-write consistency}\\(using quorums) + \vspace{1em} + \item \textbf{Fast}, no single bottleneck;\\ + works transparently with offline nodes + \end{itemize} + } \end{minipage} + \hspace{1em} +\end{frame} + +\begin{frame} + \frametitle{Why avoid consensus?} + Consensus can be implemented reasonably well in practice, so why avoid it? + \vspace{2em} + \begin{itemize} + \item \textbf{Software complexity:} RAFT and PAXOS are complex beasts;\\ + harder to prove, harder to reason about + \vspace{1.5em} + \item \textbf{Performance issues:} + \vspace{1em} + \begin{itemize} + \item Taking a decision may take an \textbf{arbitrary number of steps} (in adverse scenarios) + \vspace{1em} + \item The leader is a \textbf{bottleneck} for all requests;\\ + even in leaderless approaches, \textbf{all nodes must process all operations in order} + \vspace{1em} + \item Particularly \textbf{sensitive to higher latency} between nodes + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Objective: the right level of consistency for Garage} + + \underline{Constraints:} slow network (geographical distance), node unavailability/crashes\\ + \underline{Objective:} maximize availability, maintain an \emph{appropriate level of consistency}\\ + \vspace{1em} + \begin{enumerate} + \item<2-> \textbf{Weak consistency for most things}\\ + \vspace{1em} + \underline{Example:} \texttt{PutObject}\\ + \vspace{.5em} + If two clients write the same + object at the same time, one of the two is implicitly overwritten. + No need to coordinate, use a \emph{last-writer-wins register}. + \vspace{1em} + \item<3-> \textbf{Stronger consistency only when necessary}\\ + \vspace{1em} + \underline{Example:} \texttt{CreateBucket}\\ + \vspace{.5em} + A bucket is a reserved name in a shared namespace, + two clients should be prevented from both creating the same bucket + (\emph{mutual exclusion}). + \end{enumerate} +\end{frame} + +\begin{frame} + \frametitle{The possibility of \emph{leaderless consensus}} + Currently, Garage \emph{only has weak consistency}. Is fast, but \texttt{CreateBucket} is broken! + + \visible<2->{ + \vspace{1em} + Leaderless consensus (Antoniadis et al., 2023) alleviates issues with RAFT and PAXOS: + \vspace{1em} + \begin{itemize} + \item \textbf{No leader.} All nodes participate equally at each time step, + and different nodes can be unavailable at different times without issues. + \\ \vspace{.5em} $\to$ better tolerance to the high latency (remove bottleneck issue) + \\ $\to$ tolerates crash transparently + \vspace{1em} + \item \textbf{Simpler formalization.} The algorithm is very simple to express and to analyze in mathematical terms. + \end{itemize} + } + \visible<3->{ + \vspace{1em} + One of the possible subjects for this PhD: + \\$\to$ \emph{integration of leaderless consensus in Garage} + testing + perf eval, etc. + } +\end{frame} + +\begin{frame} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-02-03-fosdem/.gitignore b/doc/talks/2024-02-03-fosdem/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2024-02-03-fosdem/Makefile b/doc/talks/2024-02-03-fosdem/Makefile new file mode 100644 index 00000000..f0aae6a8 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/Makefile @@ -0,0 +1,19 @@ +ASSETS=../assets/lattice/lattice1.pdf_tex \ + ../assets/lattice/lattice2.pdf_tex \ + ../assets/lattice/lattice3.pdf_tex \ + ../assets/lattice/lattice4.pdf_tex \ + ../assets/lattice/lattice5.pdf_tex \ + ../assets/lattice/lattice6.pdf_tex \ + ../assets/lattice/lattice7.pdf_tex \ + ../assets/lattice/lattice8.pdf_tex \ + ../assets/logos/deuxfleurs.pdf \ + ../assets/timeline-22-24.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-02-03-fosdem/talk.pdf b/doc/talks/2024-02-03-fosdem/talk.pdf new file mode 100644 index 00000000..ef3f5c81 Binary files /dev/null and b/doc/talks/2024-02-03-fosdem/talk.pdf differ diff --git a/doc/talks/2024-02-03-fosdem/talk.tex b/doc/talks/2024-02-03-fosdem/talk.tex new file mode 100644 index 00000000..b36bf893 --- /dev/null +++ b/doc/talks/2024-02-03-fosdem/talk.tex @@ -0,0 +1,764 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage, the low-tech storage platform for geo-distributed clusters} +\author{Alex Auvolat, Deuxfleurs} +\date{FOSDEM'24, 2024-02-03} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Building a resilient system with cheap stuff} + + \only<1,4-7>{ + \begin{itemize} + \item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<4->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<6->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<7-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/atuin.jpg} + \end{center} + } + \only<8>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Object storage: a crucial component} + \begin{center} + \includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{1em} + \begin{itemize} + \item<2> Maps well to CRDT data types + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + + +% ======================================== TIMELINE +% ======================================== TIMELINE +% ======================================== TIMELINE + +\section{Recent developments} + +% ====================== v0.7.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{April 2022 - Garage v0.7.0} + Focus on \underline{observability and ecosystem integration} + \vspace{2em} + \begin{itemize} + \item \textbf{Monitoring:} metrics and traces, using OpenTelemetry + \vspace{1em} + \item Replication modes with 1 or 2 copies / weaker consistency + \vspace{1em} + \item Kubernetes integration for node discovery + \vspace{1em} + \item Admin API (v0.7.2) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Metrics (Prometheus + Grafana)} + \begin{center} + \includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Traces (Jaeger)} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ====================== v0.8.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{November 2022 - Garage v0.8.0} + Focus on \underline{performance} + \vspace{2em} + \begin{itemize} + \item \textbf{Alternative metadata DB engines} (LMDB, Sqlite) + \vspace{1em} + \item \textbf{Performance improvements:} block streaming, various optimizations... + \vspace{1em} + \item Bucket quotas (max size, max \#objects) + \vspace{1em} + \item Quality of life improvements, observability, etc. + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{About metadata DB engines} + \textbf{Issues with Sled:} + \vspace{1em} + \begin{itemize} + \item Huge files on disk + \vspace{.5em} + \item Unpredictable performance, especially on HDD + \vspace{.5em} + \item API limitations + \vspace{.5em} + \item Not actively maintained + \end{itemize} + + \vspace{2em} + \textbf{LMDB:} very stable, good performance, file size is reasonable\\ + \textbf{Sqlite} also available as a second choice + + \vspace{1em} + Sled will be removed in Garage v1.0 +\end{frame} + +\begin{frame} + \frametitle{DB engine performance comparison} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/perf/db_engine.png} + \end{center} + NB: Sqlite was slow due to synchronous mode, now configurable +\end{frame} + +\begin{frame} + \frametitle{Block streaming} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-1.png}} + \only<2>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-2.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{TTFB benchmark} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/ttfb.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Throughput benchmark} + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/perf/io-0.7-0.8-minio.png} + \end{center} +\end{frame} + +% ====================== v0.9.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.9.0} + Focus on \underline{streamlining \& usability} + \vspace{2em} + \begin{itemize} + \item Support multiple HDDs per node + \vspace{1em} + \item S3 compatibility: + \vspace{1em} + \begin{itemize} + \item support basic lifecycle configurations + \vspace{.5em} + \item allow for multipart upload part retries + \end{itemize} + \vspace{1em} + \item LMDB by default, deprecation of Sled + \vspace{1em} + \item New layout computation algorithm + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.10.0 beta} + Focus on \underline{consistency} + \vspace{2em} + \begin{itemize} + \item Fix consistency issues when reshuffling data + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Working with weak consistency} + Not using consensus limits us to the following: + \vspace{2em} + \begin{itemize} + \item<2-> \textbf{Conflict-free replicated data types} (CRDT)\\ + \vspace{1em} + {\footnotesize Non-transactional key-value stores such as S3 are equivalent to a simple CRDT:\\ + a map of \textbf{last-writer-wins registers} (each key is its own CRDT)} + \vspace{1.5em} + \item<3-> \textbf{Read-after-write consistency}\\ + \vspace{1em} + {\footnotesize Can be implemented using quorums on read and write operations} + \end{itemize} +\end{frame} + +\begin{frame}[t] + \frametitle{CRDT read-after-write consistency using quorums} + + \vspace{1em} + {\small + \textbf{Property:} If client 1 did an operation $write(x)$ and received an OK response,\\ + \hspace{2cm} and client 2 starts an operation $read()$ after client 1 received OK,\\ + \hspace{2cm} then client 2 will read a value $x' \sqsupseteq x$. + } + + \vspace{1.5em} + \begin{overprint} + \onslide<2-9> + \begin{figure} + \centering + \footnotesize + \def\svgwidth{.7\textwidth} + \only<2>{\import{../assets/lattice/}{lattice1.pdf_tex}}% + \only<3>{\import{../assets/lattice/}{lattice2.pdf_tex}}% + \only<4>{\import{../assets/lattice/}{lattice3.pdf_tex}}% + \only<5>{\import{../assets/lattice/}{lattice4.pdf_tex}}% + \only<6>{\import{../assets/lattice/}{lattice5.pdf_tex}}% + \only<7>{\import{../assets/lattice/}{lattice6.pdf_tex}}% + \only<8>{\import{../assets/lattice/}{lattice7.pdf_tex}}% + \only<9>{\import{../assets/lattice/}{lattice8.pdf_tex}}% + \end{figure} + + \onslide<10> + \begin{minipage}{.10\textwidth} + ~ + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \textbf{Algorithm $write(x)$:} + \begin{enumerate} + \item Broadcast $write(x)$ to all nodes + \item Wait for $k > n/2$ nodes to reply OK + \item Return OK + \end{enumerate} + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \vspace{1em} + \textbf{Algorithm $read()$:} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply\\ + with values $x_1, \dots, x_k$ + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + \end{minipage} + \end{overprint} +\end{frame} + +\begin{frame} + \frametitle{A hard problem: layout changes} + \begin{itemize} + \item We rely on quorums $k > n/2$ within each partition:\\ + $$n=3,~~~~~~~k\ge 2$$ + \item<2-> When rebalancing, the set of nodes responsible for a partition can change:\\ + + \vspace{1em} + \begin{minipage}{.04\linewidth}~ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{Crimson}{df-ymk} & Abricot & \textcolor{Crimson}{Courgette} \\ + \hline + Partition 1 & Ananas & \textcolor{Crimson}{Courgette} & \textcolor{Crimson}{df-ykl} \\ + \hline + Partition 2 & \textcolor{Crimson}{df-ymf} & \textcolor{Crimson}{Celeri} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + \begin{minipage}{.04\linewidth} + $\to$ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{ForestGreen}{Dahlia} & Abricot & \textcolor{ForestGreen}{Eucalyptus} \\ + \hline + Partition 1 & Ananas & \textcolor{ForestGreen}{Euphorbe} & \textcolor{ForestGreen}{Doradille} \\ + \hline + Partition 2 & \textcolor{ForestGreen}{Dahlia} & \textcolor{ForestGreen}{Echinops} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + + \vspace{2em} + \item<3-> During the rebalancing, new nodes don't yet have the data,\\ + ~~~~~~~~~~~~~~~~~~~and old nodes want to get rid of the data to free up space\\ + \vspace{1.2em} + $\to$ risk of inconsistency, \textbf{how to coordinate?} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Handling layout changes without losing consistency} + \begin{minipage}{.55\textwidth} + \begin{itemize} + \item \textbf{Solution:}\\ + \vspace{.5em} + \begin{itemize} + \item keep track of data transfer to new nodes + \vspace{.5em} + \item use multiple write quorums\\ + (new nodes + old nodes\\ + while data transfer is in progress) + \vspace{.5em} + \item switching reads to new nodes\\ + only once copy is finished + \end{itemize} + \vspace{1em} + \item \textbf{Implemented} in v0.10 + \vspace{1em} + \item \textbf{Validated} with Jepsen testing + \end{itemize} + \end{minipage} + \begin{minipage}{.23\textwidth} + \includegraphics[width=3cm]{../assets/jepsen-0.9.png}\\ + {\footnotesize Garage v0.9.0} + \end{minipage} + \begin{minipage}{.2\textwidth} + \includegraphics[width=3cm]{../assets/jepsen-0.10.png}\\ + {\footnotesize Garage v0.10 beta} + \end{minipage} +\end{frame} + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Towards v1.0...} + Focus on \underline{security \& stability} + \vspace{2em} + \begin{itemize} + \item \textbf{Security audit} in progress by Radically Open Security + \vspace{1em} + \item Misc. S3 features (SSE-C, ...) and compatibility fixes + \vspace{1em} + \item Improve UX + \vspace{1em} + \item Fix bugs + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{...and beyond!} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/survey_requested_features.png} + \end{center} +\end{frame} + +% ======================================== OPERATING +% ======================================== OPERATING +% ======================================== OPERATING + + +\section{Operating big Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{../assets/garage.drawio.pdf}}% + \only<2>{\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/2024-02-29-capitoul/.gitignore b/doc/talks/2024-02-29-capitoul/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2024-02-29-capitoul/Makefile b/doc/talks/2024-02-29-capitoul/Makefile new file mode 100644 index 00000000..aa9c35af --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/Makefile @@ -0,0 +1,10 @@ +ASSETS=../assets/logos/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-02-29-capitoul/talk.pdf b/doc/talks/2024-02-29-capitoul/talk.pdf new file mode 100644 index 00000000..33a63e23 Binary files /dev/null and b/doc/talks/2024-02-29-capitoul/talk.pdf differ diff --git a/doc/talks/2024-02-29-capitoul/talk.tex b/doc/talks/2024-02-29-capitoul/talk.tex new file mode 100644 index 00000000..95122b24 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/talk.tex @@ -0,0 +1,543 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\author{Alex Auvolat, Deuxfleurs} +\date{Capitoul, 2024-02-29} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Our very low-tech infrastructure} + + \only<1,3-6>{ + \begin{itemize} + \item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<3->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<5->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<6-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<7>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec (e.g.~locks) + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +% --------- BASED ON CRDTS ---------- + +\section{Principle 1: based on CRDTs} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{.5em} + \begin{itemize} + \item<2-> Maps well to CRDT data types + \item<3> Read-after-write consistency with quorums + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + +% --------- GEO-DISTRIBUTED MODEL ---------- + +\section{Principle 2: geo-distributed data model} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \textbf{Two keys:} + \begin{itemize} + \item Partition key: used to divide data into partitions {\small (a.k.a.~shards)} + \item Sort key: used to identify items inside a partition + \end{itemize} + + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|l|p{3cm}|} + \hline + \textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & \texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + +\begin{frame} + \frametitle{The relationship between \emph{partition} and \emph{partition key}} + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} + \vspace{1em} + \textbf{To read or write an item:} hash partition key + \\ \hspace{5cm} $\to$ determine partition number (first 8 bits) + \\ \hspace{5cm} $\to$ find associated nodes +\end{frame} + +\begin{frame} + \frametitle{Garage's internal data structures} + \centering + \includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf} +\end{frame} + +% ---------- OPERATING GARAGE --------- + +\section{Operating Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Background synchronization} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Monitoring with Prometheus + Grafana} + \begin{center} + \includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Debugging with traces} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ---------- SCALING GARAGE --------- + +\section{Scaling Garage clusters} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/assets/.gitignore b/doc/talks/assets/.gitignore new file mode 100644 index 00000000..9ce0997d --- /dev/null +++ b/doc/talks/assets/.gitignore @@ -0,0 +1,8 @@ +# Files that are auto-generated when building pdfs +deuxfleurs.pdf +timeline-22-24.pdf +lattice*.pdf_tex +lattice*.pdf + +# tmp files generated by krita +*~ diff --git a/doc/talks/assets/alex.jpg b/doc/talks/assets/alex.jpg new file mode 100644 index 00000000..eac0f0a9 Binary files /dev/null and b/doc/talks/assets/alex.jpg differ diff --git a/doc/talks/assets/atuin.jpg b/doc/talks/assets/atuin.jpg new file mode 100644 index 00000000..f2fbd61d Binary files /dev/null and b/doc/talks/assets/atuin.jpg differ diff --git a/doc/talks/assets/garage.drawio.pdf b/doc/talks/assets/garage.drawio.pdf new file mode 100644 index 00000000..a54a163c Binary files /dev/null and b/doc/talks/assets/garage.drawio.pdf differ diff --git a/doc/talks/assets/garage.drawio.png b/doc/talks/assets/garage.drawio.png new file mode 100644 index 00000000..386dd862 Binary files /dev/null and b/doc/talks/assets/garage.drawio.png differ diff --git a/doc/talks/assets/garage2.drawio.png b/doc/talks/assets/garage2.drawio.png new file mode 100644 index 00000000..8562fbcf Binary files /dev/null and b/doc/talks/assets/garage2.drawio.png differ diff --git a/doc/talks/assets/garage2a.drawio.pdf b/doc/talks/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..422c9343 Binary files /dev/null and b/doc/talks/assets/garage2a.drawio.pdf differ diff --git a/doc/talks/assets/garage2b.drawio.pdf b/doc/talks/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..05a9710e Binary files /dev/null and b/doc/talks/assets/garage2b.drawio.pdf differ diff --git a/doc/talks/assets/garage_sync.drawio.pdf b/doc/talks/assets/garage_sync.drawio.pdf new file mode 100644 index 00000000..a94b3572 Binary files /dev/null and b/doc/talks/assets/garage_sync.drawio.pdf differ diff --git a/doc/talks/assets/garage_sync.drawio.png b/doc/talks/assets/garage_sync.drawio.png new file mode 100644 index 00000000..2e7b5af0 Binary files /dev/null and b/doc/talks/assets/garage_sync.drawio.png differ diff --git a/doc/talks/assets/garage_tables.pdf b/doc/talks/assets/garage_tables.pdf new file mode 100644 index 00000000..3c54cdce Binary files /dev/null and b/doc/talks/assets/garage_tables.pdf differ diff --git a/doc/talks/assets/geodistrib_paper.png b/doc/talks/assets/geodistrib_paper.png new file mode 100644 index 00000000..318d8e55 Binary files /dev/null and b/doc/talks/assets/geodistrib_paper.png differ diff --git a/doc/talks/assets/inframap_jdll2023.pdf b/doc/talks/assets/inframap_jdll2023.pdf new file mode 100644 index 00000000..77f97baa Binary files /dev/null and b/doc/talks/assets/inframap_jdll2023.pdf differ diff --git a/doc/talks/assets/intro/slide1.png b/doc/talks/assets/intro/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/assets/intro/slide1.png differ diff --git a/doc/talks/assets/intro/slide2.png b/doc/talks/assets/intro/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/assets/intro/slide2.png differ diff --git a/doc/talks/assets/intro/slide3.png b/doc/talks/assets/intro/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/assets/intro/slide3.png differ diff --git a/doc/talks/assets/intro/slideB1.png b/doc/talks/assets/intro/slideB1.png new file mode 100644 index 00000000..b14b6070 Binary files /dev/null and b/doc/talks/assets/intro/slideB1.png differ diff --git a/doc/talks/assets/intro/slideB2.png b/doc/talks/assets/intro/slideB2.png new file mode 100644 index 00000000..a881a796 Binary files /dev/null and b/doc/talks/assets/intro/slideB2.png differ diff --git a/doc/talks/assets/intro/slideB3.png b/doc/talks/assets/intro/slideB3.png new file mode 100644 index 00000000..830709d2 Binary files /dev/null and b/doc/talks/assets/intro/slideB3.png differ diff --git a/doc/talks/assets/intro/slides.svg b/doc/talks/assets/intro/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/assets/intro/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/assets/intro/slidesB.svg b/doc/talks/assets/intro/slidesB.svg new file mode 100644 index 00000000..c0a6e97c --- /dev/null +++ b/doc/talks/assets/intro/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database diff --git a/doc/talks/assets/jepsen-0.10.png b/doc/talks/assets/jepsen-0.10.png new file mode 100644 index 00000000..d7895e31 Binary files /dev/null and b/doc/talks/assets/jepsen-0.10.png differ diff --git a/doc/talks/assets/jepsen-0.9.png b/doc/talks/assets/jepsen-0.9.png new file mode 100644 index 00000000..3f654038 Binary files /dev/null and b/doc/talks/assets/jepsen-0.9.png differ diff --git a/doc/talks/assets/lattice/lattice1.svg b/doc/talks/assets/lattice/lattice1.svg new file mode 100644 index 00000000..8bfa5aa7 --- /dev/null +++ b/doc/talks/assets/lattice/lattice1.svg @@ -0,0 +1,433 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + diff --git a/doc/talks/assets/lattice/lattice2.svg b/doc/talks/assets/lattice/lattice2.svg new file mode 100644 index 00000000..adcd92cb --- /dev/null +++ b/doc/talks/assets/lattice/lattice2.svg @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice3.svg b/doc/talks/assets/lattice/lattice3.svg new file mode 100644 index 00000000..640dc468 --- /dev/null +++ b/doc/talks/assets/lattice/lattice3.svg @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice4.svg b/doc/talks/assets/lattice/lattice4.svg new file mode 100644 index 00000000..b2a99e28 --- /dev/null +++ b/doc/talks/assets/lattice/lattice4.svg @@ -0,0 +1,525 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice5.svg b/doc/talks/assets/lattice/lattice5.svg new file mode 100644 index 00000000..bc6b7195 --- /dev/null +++ b/doc/talks/assets/lattice/lattice5.svg @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice6.svg b/doc/talks/assets/lattice/lattice6.svg new file mode 100644 index 00000000..176b1715 --- /dev/null +++ b/doc/talks/assets/lattice/lattice6.svg @@ -0,0 +1,553 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + + diff --git a/doc/talks/assets/lattice/lattice7.svg b/doc/talks/assets/lattice/lattice7.svg new file mode 100644 index 00000000..7ce8bda8 --- /dev/null +++ b/doc/talks/assets/lattice/lattice7.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/assets/lattice/lattice8.svg b/doc/talks/assets/lattice/lattice8.svg new file mode 100644 index 00000000..c94a69b2 --- /dev/null +++ b/doc/talks/assets/lattice/lattice8.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/doc/talks/assets/logos/AGPLv3_Logo.png b/doc/talks/assets/logos/AGPLv3_Logo.png new file mode 100644 index 00000000..445284a3 Binary files /dev/null and b/doc/talks/assets/logos/AGPLv3_Logo.png differ diff --git a/doc/talks/assets/logos/Amazon-S3.jpg b/doc/talks/assets/logos/Amazon-S3.jpg new file mode 100644 index 00000000..a9501973 Binary files /dev/null and b/doc/talks/assets/logos/Amazon-S3.jpg differ diff --git a/doc/talks/assets/logos/NGI0Entrust_tag.png b/doc/talks/assets/logos/NGI0Entrust_tag.png new file mode 100644 index 00000000..064c4aee Binary files /dev/null and b/doc/talks/assets/logos/NGI0Entrust_tag.png differ diff --git a/doc/talks/assets/logos/NGI0Entrust_tag.svg b/doc/talks/assets/logos/NGI0Entrust_tag.svg new file mode 100644 index 00000000..ca0d2814 --- /dev/null +++ b/doc/talks/assets/logos/NGI0Entrust_tag.svg @@ -0,0 +1,149 @@ + + + +image/svg+xml + + + + + + +NGI Zero Entrust + + + + diff --git a/doc/talks/assets/logos/capitole_du_libre.png b/doc/talks/assets/logos/capitole_du_libre.png new file mode 100644 index 00000000..eaa521af Binary files /dev/null and b/doc/talks/assets/logos/capitole_du_libre.png differ diff --git a/doc/talks/assets/logos/deuxfleurs.svg b/doc/talks/assets/logos/deuxfleurs.svg new file mode 100644 index 00000000..c298c22b --- /dev/null +++ b/doc/talks/assets/logos/deuxfleurs.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + D + F + diff --git a/doc/talks/assets/logos/fosdem22.png b/doc/talks/assets/logos/fosdem22.png new file mode 100644 index 00000000..17332e4b Binary files /dev/null and b/doc/talks/assets/logos/fosdem22.png differ diff --git a/doc/talks/assets/logos/fosdem24.png b/doc/talks/assets/logos/fosdem24.png new file mode 100644 index 00000000..11631695 Binary files /dev/null and b/doc/talks/assets/logos/fosdem24.png differ diff --git a/doc/talks/assets/logos/jdll.png b/doc/talks/assets/logos/jdll.png new file mode 100644 index 00000000..a7cbe091 Binary files /dev/null and b/doc/talks/assets/logos/jdll.png differ diff --git a/doc/talks/assets/logos/logo_chatons.png b/doc/talks/assets/logos/logo_chatons.png new file mode 100644 index 00000000..890cf17e Binary files /dev/null and b/doc/talks/assets/logos/logo_chatons.png differ diff --git a/doc/talks/assets/logos/minio.png b/doc/talks/assets/logos/minio.png new file mode 100644 index 00000000..a71e9ccc Binary files /dev/null and b/doc/talks/assets/logos/minio.png differ diff --git a/doc/talks/assets/logos/ngi-pointer.png b/doc/talks/assets/logos/ngi-pointer.png new file mode 100644 index 00000000..730ececa Binary files /dev/null and b/doc/talks/assets/logos/ngi-pointer.png differ diff --git a/doc/talks/assets/logos/nlnet.svg b/doc/talks/assets/logos/nlnet.svg new file mode 100644 index 00000000..373c8d8f --- /dev/null +++ b/doc/talks/assets/logos/nlnet.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/talks/assets/logos/pses.png b/doc/talks/assets/logos/pses.png new file mode 100644 index 00000000..da871f83 Binary files /dev/null and b/doc/talks/assets/logos/pses.png differ diff --git a/doc/talks/assets/logos/rust_logo.png b/doc/talks/assets/logos/rust_logo.png new file mode 100644 index 00000000..0e4809ec Binary files /dev/null and b/doc/talks/assets/logos/rust_logo.png differ diff --git a/doc/talks/assets/map.png b/doc/talks/assets/map.png new file mode 100644 index 00000000..1dff3ab6 Binary files /dev/null and b/doc/talks/assets/map.png differ diff --git a/doc/talks/assets/neptune.jpg b/doc/talks/assets/neptune.jpg new file mode 100644 index 00000000..61fcbff6 Binary files /dev/null and b/doc/talks/assets/neptune.jpg differ diff --git a/doc/talks/assets/perf/db_engine.png b/doc/talks/assets/perf/db_engine.png new file mode 100644 index 00000000..b1124b04 Binary files /dev/null and b/doc/talks/assets/perf/db_engine.png differ diff --git a/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png b/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png new file mode 100644 index 00000000..92eac3fb Binary files /dev/null and b/doc/talks/assets/perf/endpoint_latency_0.7_0.8_minio.png differ diff --git a/doc/talks/assets/perf/io-0.7-0.8-minio.png b/doc/talks/assets/perf/io-0.7-0.8-minio.png new file mode 100644 index 00000000..f581a22e Binary files /dev/null and b/doc/talks/assets/perf/io-0.7-0.8-minio.png differ diff --git a/doc/talks/assets/perf/ttfb.png b/doc/talks/assets/perf/ttfb.png new file mode 100644 index 00000000..c0335bd9 Binary files /dev/null and b/doc/talks/assets/perf/ttfb.png differ diff --git a/doc/talks/assets/schema-streaming-1.png b/doc/talks/assets/schema-streaming-1.png new file mode 100644 index 00000000..4113f030 Binary files /dev/null and b/doc/talks/assets/schema-streaming-1.png differ diff --git a/doc/talks/assets/schema-streaming-2.png b/doc/talks/assets/schema-streaming-2.png new file mode 100644 index 00000000..2f97bb33 Binary files /dev/null and b/doc/talks/assets/schema-streaming-2.png differ diff --git a/doc/talks/assets/schema-streaming.png b/doc/talks/assets/schema-streaming.png new file mode 100644 index 00000000..3b24d910 Binary files /dev/null and b/doc/talks/assets/schema-streaming.png differ diff --git a/doc/talks/assets/screenshots/garage_stats_0.10.png b/doc/talks/assets/screenshots/garage_stats_0.10.png new file mode 100644 index 00000000..78a36ead Binary files /dev/null and b/doc/talks/assets/screenshots/garage_stats_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.10.png b/doc/talks/assets/screenshots/garage_status_0.10.png new file mode 100644 index 00000000..510717ca Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.9_prod.png b/doc/talks/assets/screenshots/garage_status_0.9_prod.png new file mode 100644 index 00000000..e1a05899 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.9_prod.png differ diff --git a/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png b/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png new file mode 100644 index 00000000..99d46f98 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_0.9_prod_zonehl.png differ diff --git a/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png b/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png new file mode 100644 index 00000000..b167687d Binary files /dev/null and b/doc/talks/assets/screenshots/garage_status_unhealthy_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_worker_list_0.10.png b/doc/talks/assets/screenshots/garage_worker_list_0.10.png new file mode 100644 index 00000000..078f0607 Binary files /dev/null and b/doc/talks/assets/screenshots/garage_worker_list_0.10.png differ diff --git a/doc/talks/assets/screenshots/garage_worker_param_0.10.png b/doc/talks/assets/screenshots/garage_worker_param_0.10.png new file mode 100644 index 00000000..39ee11aa Binary files /dev/null and b/doc/talks/assets/screenshots/garage_worker_param_0.10.png differ diff --git a/doc/talks/assets/screenshots/grafana_dashboard.png b/doc/talks/assets/screenshots/grafana_dashboard.png new file mode 100644 index 00000000..bda2d732 Binary files /dev/null and b/doc/talks/assets/screenshots/grafana_dashboard.png differ diff --git a/doc/talks/assets/screenshots/jaeger_listobjects.png b/doc/talks/assets/screenshots/jaeger_listobjects.png new file mode 100644 index 00000000..ab7d68f4 Binary files /dev/null and b/doc/talks/assets/screenshots/jaeger_listobjects.png differ diff --git a/doc/talks/assets/screenshots/jaeger_putobject.png b/doc/talks/assets/screenshots/jaeger_putobject.png new file mode 100644 index 00000000..7966275e Binary files /dev/null and b/doc/talks/assets/screenshots/jaeger_putobject.png differ diff --git a/doc/talks/assets/survey_requested_features.png b/doc/talks/assets/survey_requested_features.png new file mode 100644 index 00000000..07247c46 Binary files /dev/null and b/doc/talks/assets/survey_requested_features.png differ diff --git a/doc/talks/assets/timeline-22-24.svg b/doc/talks/assets/timeline-22-24.svg new file mode 100644 index 00000000..90fe5de4 --- /dev/null +++ b/doc/talks/assets/timeline-22-24.svg @@ -0,0 +1,327 @@ + + + +Feb 6, 2022Feb 3, 2024v0.6.0Feb 2, 2022v0.7.0Apr 8, 2022v0.8.0Nov 21, 2022v0.9.0Oct 10, 2023v0.10.0 betaTowards v1.0 ?Apr/May, 2024Nov 19, 2022Apr 1, 2023PSESJun 6, 2023 diff --git a/doc/talks/shell.nix b/doc/talks/shell.nix index 161a61e1..bbee0714 100644 --- a/doc/talks/shell.nix +++ b/doc/talks/shell.nix @@ -5,7 +5,7 @@ let scheme-basic beamer amsmath mathtools breqn environ - multirow graphics import adjustbox tabu vwcol stmaryrd ulem ragged2e + multirow graphics import adjustbox tabu vwcol stmaryrd ulem ragged2e textpos dvisvgm dvipng wrapfig hyperref capt-of; }); in pkgs.mkShell { nativeBuildInputs = [ pkgs.gnumake latex ]; } diff --git a/flake.lock b/flake.lock index beba6a7b..211b70e0 100644 --- a/flake.lock +++ b/flake.lock @@ -1,67 +1,45 @@ { "nodes": { - "cargo2nix": { - "inputs": { - "flake-compat": "flake-compat", - "flake-utils": "flake-utils", - "nixpkgs": [ - "nixpkgs" - ], - "rust-overlay": "rust-overlay" - }, + "crane": { "locked": { - "lastModified": 1666087781, - "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=", - "owner": "Alexis211", - "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "lastModified": 1737689766, + "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", + "owner": "ipetkov", + "repo": "crane", + "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", "type": "github" }, "original": { - "owner": "Alexis211", - "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "owner": "ipetkov", + "repo": "crane", "type": "github" } }, "flake-compat": { - "flake": false, "locked": { - "lastModified": 1650374568, - "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", - "owner": "edolstra", + "lastModified": 1717312683, + "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=", + "owner": "nix-community", "repo": "flake-compat", - "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea", "type": "github" }, "original": { - "owner": "edolstra", + "owner": "nix-community", "repo": "flake-compat", "type": "github" } }, "flake-utils": { - "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", - "type": "github" + "inputs": { + "systems": "systems" }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -72,49 +50,62 @@ }, "nixpkgs": { "locked": { - "lastModified": 1665657542, - "narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=", + "lastModified": 1763977559, + "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3073c49bc0163fea6a121c276f526837672b555", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3073c49bc0163fea6a121c276f526837672b555", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" } }, "root": { "inputs": { - "cargo2nix": "cargo2nix", - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "crane": "crane", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" } }, "rust-overlay": { "inputs": { - "flake-utils": [ - "cargo2nix", - "flake-utils" - ], "nixpkgs": [ - "cargo2nix", "nixpkgs" ] }, "locked": { - "lastModified": 1664247556, - "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=", + "lastModified": 1763952169, + "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" }, "original": { "owner": "oxalica", "repo": "rust-overlay", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", "type": "github" } } diff --git a/flake.nix b/flake.nix index 7eb9e33b..01a077c4 100644 --- a/flake.nix +++ b/flake.nix @@ -2,35 +2,95 @@ description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; + # Nixpkgs 25.05 as of 2025-11-24 inputs.nixpkgs.url = - "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555"; - inputs.cargo2nix = { - # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection - url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; - inputs.nixpkgs.follows = "nixpkgs"; - }; + "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632"; + + # Rust overlay as of 2025-11-24 + inputs.rust-overlay.url = + "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b"; + inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; + + inputs.crane.url = "github:ipetkov/crane"; + + inputs.flake-compat.url = "github:nix-community/flake-compat"; inputs.flake-utils.url = "github:numtide/flake-utils"; - outputs = { self, nixpkgs, cargo2nix, flake-utils }: + outputs = { self, nixpkgs, flake-utils, crane, rust-overlay, ... }: let - git_version = self.lastModifiedDate; compile = import ./nix/compile.nix; - in flake-utils.lib.eachDefaultSystem (system: - let pkgs = nixpkgs.legacyPackages.${system}; - in { - packages = { - default = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; - release = true; - }).workspace.garage { compileMode = "build"; }; - }; - devShell = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; + in + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + packageFor = target: release: (compile { + inherit system target nixpkgs crane rust-overlay release; + }).garage; + testWith = extraTestEnv: (compile { + inherit system nixpkgs crane rust-overlay extraTestEnv; release = false; - }).workspaceShell { packages = [ pkgs.rustfmt ]; }; + }).garage-test; + lints = (compile { + inherit system nixpkgs crane rust-overlay; + release = false; + }); + in + { + packages = { + # default = native release build + default = packageFor null true; + + # = cross-compiled, statically-linked release builds + amd64 = packageFor "x86_64-unknown-linux-musl" true; + i386 = packageFor "i686-unknown-linux-musl" true; + arm64 = packageFor "aarch64-unknown-linux-musl" true; + arm = packageFor "armv6l-unknown-linux-musl" true; + + # dev = native dev build + dev = packageFor null false; + + # test = cargo test + tests = testWith {}; + tests-lmdb = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "lmdb"; + }; + tests-sqlite = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite"; + }; + tests-fjall = testWith { + GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall"; + }; + + # lints (fmt, clippy) + fmt = lints.garage-cargo-fmt; + clippy = lints.garage-cargo-clippy; + }; + + # ---- developpment shell, for making native builds only ---- + devShells = + let + targets = compile { + inherit system nixpkgs crane rust-overlay; + }; + in + { + default = targets.devShell; + + # import the full shell using `nix develop .#full` + full = pkgs.mkShell { + buildInputs = with pkgs; [ + targets.toolchain + protobuf + clang + mold + # ---- extra packages for dev tasks ---- + rust-analyzer + cargo-audit + cargo-outdated + cargo-machete + nixpkgs-fmt + ]; + }; + }; }); } diff --git a/k2v_test.py b/k2v_test.py deleted file mode 100755 index 3219056e..00000000 --- a/k2v_test.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -import os -import requests -from datetime import datetime - -# let's talk to our AWS Elasticsearch cluster -#from requests_aws4auth import AWS4Auth -#auth = AWS4Auth('GK31c2f218a2e44f485b94239e', -# 'b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', -# 'us-east-1', -# 's3') - -from aws_requests_auth.aws_auth import AWSRequestsAuth -auth = AWSRequestsAuth(aws_access_key='GK31c2f218a2e44f485b94239e', - aws_secret_access_key='b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', - aws_host='localhost:3812', - aws_region='us-east-1', - aws_service='k2v') - - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - - -sort_keys = ["a", "b", "c", "d"] - -for sk in sort_keys: - print("-- (%s) Put initial (no CT)"%sk) - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - data='{}: Hello, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- ReadIndex") - response = requests.get('http://localhost:3812/alex', - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put with CT") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Good bye, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put again with same CT (concurrent)") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Concurrent value, oops'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- Delete") - response = requests.delete('http://localhost:3812/alex/root?sort_key=%s'%sk, - headers={'x-garage-causality-token': ct}, - auth=auth) - print(response.headers) - print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -print("-- InsertBatch") -response = requests.post('http://localhost:3812/alex', - auth=auth, - data=''' -[ - {"pk": "root", "sk": "a", "ct": null, "v": "aW5pdGlhbCB0ZXN0Cg=="}, - {"pk": "root", "sk": "b", "ct": null, "v": "aW5pdGlhbCB0ZXN1Cg=="}, - {"pk": "root", "sk": "c", "ct": null, "v": "aW5pdGlhbCB0ZXN2Cg=="} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"}, - {"partitionKey": "root", "tombstones": true}, - {"partitionKey": "root", "tombstones": true, "limit": 2}, - {"partitionKey": "root", "start": "c", "singleItem": true}, - {"partitionKey": "root", "start": "b", "end": "d", "tombstones": true} -] -''') -print(response.headers) -print(response.text) - - -print("-- DeleteBatch") -response = requests.post('http://localhost:3812/alex?delete', - auth=auth, - data=''' -[ - {"partitionKey": "root", "start": "b", "end": "c"} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"} -] -''') -print(response.headers) -print(response.text) diff --git a/nix/build_index.nix b/nix/build_index.nix index 7cc4f62c..7869566f 100644 --- a/nix/build_index.nix +++ b/nix/build_index.nix @@ -2,7 +2,7 @@ with import ./common.nix; let - pkgs = import pkgsSrc { }; + pkgs = import nixpkgs { }; lib = pkgs.lib; /* Converts a key list and a value list to a set diff --git a/nix/common.nix b/nix/common.nix index 57f354dd..2c03f04e 100644 --- a/nix/common.nix +++ b/nix/common.nix @@ -1,19 +1,18 @@ -rec { - # * Fixed dependencies - pkgsSrc = fetchTarball { - # As of 2022-10-13 - url = - "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip"; - sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s"; - }; - cargo2nixSrc = fetchGit { - # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection - url = "https://github.com/Alexis211/cargo2nix"; - ref = "custom_unstable"; - rev = "a7a61179b66054904ef6a195d8da736eaaa06c36"; +let + lock = builtins.fromJSON (builtins.readFile ../flake.lock); + + inherit (lock.nodes.flake-compat.locked) owner repo rev narHash; + + flake-compat = fetchTarball { + url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; + sha256 = narHash; }; - # * Shared objects - cargo2nix = import cargo2nixSrc; - cargo2nixOverlay = cargo2nix.overlays.default; + flake = (import flake-compat { system = builtins.currentSystem; src = ../.; }); +in + +{ + flake = flake.defaultNix; + nixpkgs = flake.defaultNix.inputs.nixpkgs; + devShells = flake.defaultNix.devShells.${builtins.currentSystem}; } diff --git a/nix/compile.nix b/nix/compile.nix index 54e920db..c6df9dbd 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -1,185 +1,64 @@ -{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc" -, release ? false, git_version ? null, features ? null, }: +{ + /* build inputs */ + nixpkgs, + crane, + rust-overlay, + + /* parameters */ + system, + git_version ? null, + target ? null, + release ? false, + features ? null, + extraTestEnv ? {} +}: let log = v: builtins.trace v v; + # NixOS and Rust/Cargo triples do not match for ARM, fix it here. + rustTarget = if target == "armv6l-unknown-linux-musleabihf" then + "arm-unknown-linux-musleabihf" + else + target; + + rustTargetEnvMap = { + "x86_64-unknown-linux-musl" = "X86_64_UNKNOWN_LINUX_MUSL"; + "aarch64-unknown-linux-musl" = "AARCH64_UNKNOWN_LINUX_MUSL"; + "i686-unknown-linux-musl" = "I686_UNKNOWN_LINUX_MUSL"; + "arm-unknown-linux-musleabihf" = "ARM_UNKNOWN_LINUX_MUSLEABIHF"; + }; + + pkgsNative = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + pkgs = if target != null then - import pkgsSrc { + import nixpkgs { inherit system; crossSystem = { config = target; isStatic = true; }; - overlays = [ cargo2nixOverlay ]; + overlays = [ (import rust-overlay) ]; } else - import pkgsSrc { - inherit system; - overlays = [ cargo2nixOverlay ]; - }; + pkgsNative; - /* Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. - This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust - to avoid incompatibilities with time_t between different versions of musl - (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. - So we want a Rust release that is bound to our Nix repository to avoid these problems. - See here for more info: https://musl.libc.org/time64.html - Because Cargo2nix does not support the Rust environment shipped by NixOS, - we emulate the structure of the Rust object created by rustOverlay. - In practise, rustOverlay ships rustc+cargo in a single derivation while - NixOS ships them in separate ones. We reunite them with symlinkJoin. - */ - toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl" - || target == "aarch64-unknown-linux-musl" then { - rustVersion = "1.63.0"; - extraRustComponents = [ "clippy" ]; - } else { - rustToolchain = pkgs.symlinkJoin { - name = "rust-static-toolchain-${target}"; - paths = [ - pkgs.rustPlatform.rust.cargo - pkgs.rustPlatform.rust.rustc - # clippy not needed, it only runs on amd64 - ]; - }; - }; + inherit (pkgs) lib stdenv; - buildEnv = (drv: - { - rustc = drv.setBuildEnv; - clippy = '' - ${drv.setBuildEnv or ""} - echo - echo --- BUILDING WITH CLIPPY --- - echo - - export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" - export RUSTC="''${CLIPPY_DRIVER}" - ''; - }.${compiler}); - - /* Cargo2nix provides many overrides by default, you can take inspiration from them: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - - You can have a complete list of the available options by looking at the overriden object, mkcrate: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix - */ - packageOverrides = pkgs: - pkgs.rustBuilder.overrides.all ++ [ - /* [1] We add some logic to compile our crates with clippy, it provides us many additional lints - - [2] We need to alter Nix hardening to make static binaries: PIE, - Position Independent Executables seems to be supported only on amd64. Having - this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. - Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets - (only amd64 curently) through the `-static-pie` flag. - PIE is a feature used by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - - [3] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. - - [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. - However the features to do so get activated for some reason (due to a bug in cargo2nix?), - so disable them manually here. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; - overrideAttrs = drv: - (if git_version != null then { - # [3] - preConfigure = '' - ${drv.preConfigure or ""} - export GIT_VERSION="${git_version}" - ''; - } else - { }) // { - # [1] - setBuildEnv = (buildEnv drv); - # [2] - hardeningDisable = [ "pie" ]; - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_db"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_util"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_table"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_block"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_model"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_api"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_web"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "k2v-client"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "libsodium-sys"; - overrideArgs = old: { - features = [ ]; # [4] - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "zstd-sys"; - overrideArgs = old: { - features = [ ]; # [4] - }; - }) + toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override { + targets = lib.optionals (target != null) [ rustTarget ]; + extensions = [ + "rust-src" + "rustfmt" ]; + }); + + craneLib = (crane.mkLib pkgs).overrideToolchain toolchainFn; + + src = craneLib.cleanCargoSource ../.; /* We ship some parts of the code disabled by default by putting them behind a flag. It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). @@ -189,17 +68,16 @@ let rootFeatures = if features != null then features else - ([ "garage/bundled-libs" "garage/sled" "garage/k2v" ] ++ (if release then [ - "garage/consul-discovery" - "garage/kubernetes-discovery" - "garage/metrics" - "garage/telemetry-otlp" - "garage/lmdb" - "garage/sqlite" - ] else - [ ])); + ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [ + "consul-discovery" + "kubernetes-discovery" + "metrics" + "telemetry-otlp" + "syslog" + "journald" + ])); - packageFun = import ../Cargo.nix; + featuresStr = lib.concatStringsSep "," rootFeatures; /* We compile fully static binaries with musl to simplify deployment on most systems. When possible, we reactivate PIE hardening (see above). @@ -210,12 +88,9 @@ let For more information on static builds, please refer to Rust's RFC 1721. https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage */ - - codegenOpts = { - "armv6l-unknown-linux-musleabihf" = [ - "target-feature=+crt-static" - "link-arg=-static" - ]; # compile as dynamic with static-pie + codegenOptsMap = { + "x86_64-unknown-linux-musl" = + [ "target-feature=+crt-static" "link-arg=-static-pie" ]; "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" @@ -224,17 +99,106 @@ let "target-feature=+crt-static" "link-arg=-static" ]; # segfault with static-pie - "x86_64-unknown-linux-musl" = - [ "target-feature=+crt-static" "link-arg=-static-pie" ]; + "armv6l-unknown-linux-musleabihf" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # compile as dynamic with static-pie }; - # NixOS and Rust/Cargo triples do not match for ARM, fix it here. - rustTarget = if target == "armv6l-unknown-linux-musleabihf" then - "arm-unknown-linux-musleabihf" - else - target; + codegenOpts = if target != null then codegenOptsMap.${target} else [ + "link-arg=-fuse-ld=mold" + ]; -in pkgs.rustBuilder.makePackageSet ({ - inherit release packageFun packageOverrides codegenOpts rootFeatures; - target = rustTarget; -} // toolchainOptions) + commonArgs = + { + inherit src; + pname = "garage"; + version = "dev"; + + strictDeps = true; + cargoExtraArgs = "--locked --features ${featuresStr}"; + cargoTestExtraArgs = "--workspace"; + + nativeBuildInputs = [ + pkgsNative.protobuf + pkgs.stdenv.cc + ] ++ lib.optionals (target == null) [ + pkgs.clang + pkgs.mold + ]; + + CARGO_PROFILE = if release then "release" else "dev"; + CARGO_BUILD_RUSTFLAGS = + lib.concatStringsSep + " " + (builtins.map (flag: "-C ${flag}") codegenOpts); + } + // + (if rustTarget != null then { + CARGO_BUILD_TARGET = rustTarget; + + "CARGO_TARGET_${rustTargetEnvMap.${rustTarget}}_LINKER" = "${stdenv.cc.targetPrefix}cc"; + + HOST_CC = "${stdenv.cc.nativePrefix}cc"; + TARGET_CC = "${stdenv.cc.targetPrefix}cc"; + } else { + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "clang"; + }); + +in rec { + toolchain = toolchainFn pkgs; + + devShell = pkgs.mkShell { + buildInputs = [ + toolchain + ] ++ (with pkgs; [ + protobuf + clang + mold + ]); + }; + + # ---- building garage ---- + + garage-deps = craneLib.buildDepsOnly commonArgs; + + garage = craneLib.buildPackage (commonArgs // { + cargoArtifacts = garage-deps; + + doCheck = false; + } // + (if git_version != null then { + version = git_version; + GIT_VERSION = git_version; + } else {})); + + # ---- testing garage ---- + + garage-test-bin = craneLib.cargoBuild (commonArgs // { + cargoArtifacts = garage-deps; + + pname = "garage-tests"; + + CARGO_PROFILE = "test"; + cargoExtraArgs = "${commonArgs.cargoExtraArgs} --tests --workspace"; + doCheck = false; + }); + + garage-test = craneLib.cargoTest (commonArgs // { + cargoArtifacts = garage-test-bin; + nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ + pkgs.cacert + ]; + } // extraTestEnv); + + # ---- source code linting ---- + + garage-cargo-fmt = craneLib.cargoFmt (commonArgs // { + cargoExtraArgs = ""; + }); + + garage-cargo-clippy = craneLib.cargoClippy (commonArgs // { + cargoArtifacts = garage-deps; + cargoClippyExtraArgs = "--all-targets -- -D warnings"; + }); +} diff --git a/nix/kaniko.nix b/nix/kaniko.nix deleted file mode 100644 index 64cadd14..00000000 --- a/nix/kaniko.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "kaniko"; - version = "1.6.0"; - - src = pkgs.fetchFromGitHub { - owner = "GoogleContainerTools"; - repo = "kaniko"; - rev = "v${version}"; - sha256 = "1fnclr556avxay6pvgw5ya3xbxfnf2gv4njq2hr4fd6fcjyslq5h"; - }; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; - homepage = "https://github.com/GoogleContainerTools/kaniko"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix deleted file mode 100644 index 1090a6ef..00000000 --- a/nix/manifest-tool.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "manifest-tool"; - version = "2.0.5"; - - src = pkgs.fetchFromGitHub { - owner = "estesp"; - repo = "manifest-tool"; - rev = "v${version}"; - sha256 = "hjCGKnE0yrlnF/VIzOwcDzmQX3Wft+21KCny/opqdLg="; - } + "/v2"; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "Command line tool to create and query container image manifest list/indexes"; - homepage = "https://github.com/estesp/manifest-tool"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/toolchain.nix b/nix/toolchain.nix deleted file mode 100644 index 532db74e..00000000 --- a/nix/toolchain.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ system ? builtins.currentSystem, }: - -with import ./common.nix; - -let - platforms = [ - #"x86_64-unknown-linux-musl" - "i686-unknown-linux-musl" - #"aarch64-unknown-linux-musl" - "armv6l-unknown-linux-musleabihf" - ]; - pkgsList = builtins.map (target: - import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - }) platforms; - pkgsHost = import pkgsSrc { }; - lib = pkgsHost.lib; - kaniko = (import ./kaniko.nix) pkgsHost; - winscp = (import ./winscp.nix) pkgsHost; - manifestTool = (import ./manifest-tool.nix) pkgsHost; -in lib.flatten (builtins.map (pkgs: [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - pkgs.buildPackages.stdenv.cc -]) pkgsList) ++ [ kaniko winscp manifestTool ] - diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh index 9bcf5bda..708c2c43 100755 --- a/script/dev-bucket.sh +++ b/script/dev-bucket.sh @@ -9,11 +9,22 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" -garage -c /tmp/config.1.toml bucket create eprouvette -KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur) +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi + +$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette +if [ "$GARAGE_08" = "1" ]; then + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur) +else + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur) +fi ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'` -garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY +$GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index c7fbe08d..998ffdb9 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -11,11 +11,16 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m") export RUST_BACKTRACE=1 -export RUST_LOG=garage=info,garage_api=debug,netapp=trace +export RUST_LOG=garage=info,garage_api_common=debug,garage_api_s3=debug MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" -WHICH_GARAGE=$(which garage || exit 1) -echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "${MAIN_LABEL} Found garage at: ${GARAGE_BIN}\n" +else + echo -en "${MAIN_LABEL} Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version NETWORK_SECRET="$(openssl rand -hex 32)" @@ -28,6 +33,7 @@ LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m" cat > $CONF_PATH <&1|while read r; do echo -en "$LABEL $r\n"; done) & +($GARAGE_BIN -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & done # >>>>>>>>>>>>>>>> END FOR LOOP ON NODES @@ -73,14 +79,14 @@ fi sleep 3 # Establish connections between nodes for count in $(seq 1 3); do - NODE=$(garage -c /tmp/config.$count.toml node id -q) + NODE=$($GARAGE_BIN -c /tmp/config.$count.toml node id -q) for count2 in $(seq 1 3); do - garage -c /tmp/config.$count2.toml node connect $NODE + $GARAGE_BIN -c /tmp/config.$count2.toml node connect $NODE done done RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo -en "${MAIN_LABEL} Garage did not start" diff --git a/script/dev-configure.sh b/script/dev-configure.sh index f0a7843d..0649cdbe 100755 --- a/script/dev-configure.sh +++ b/script/dev-configure.sh @@ -9,9 +9,17 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version + sleep 5 RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo "garage did not start in time, failing." @@ -21,11 +29,20 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do sleep 1 done -garage -c /tmp/config.1.toml status \ - | grep 'NO ROLE' \ - | grep -Po '^[0-9a-f]+' \ - | while read id; do - garage -c /tmp/config.1.toml layout assign $id -z dc1 -c 1 - done +if [ "$GARAGE_08" = "1" ]; then + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1 + done +else + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G + done +fi -garage -c /tmp/config.1.toml layout apply --version 1 +$GARAGE_BIN -c /tmp/config.1.toml layout apply --version 1 diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh index 9436c2c7..41f1fdde 100644 --- a/script/dev-env-aws.sh +++ b/script/dev-env-aws.sh @@ -1,7 +1,8 @@ export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` export AWS_DEFAULT_REGION='garage' - +export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' +# FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0. function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } aws --version diff --git a/script/helm/README.md b/script/helm/README.md index 5f919a23..1b22e604 100644 --- a/script/helm/README.md +++ b/script/helm/README.md @@ -1,3 +1,3 @@ # Garage helm3 chart -Documentation is located [here](/doc/book/cookbook/kubernetes.md). +Documentation is located [here](https://garagehq.deuxfleurs.fr/documentation/cookbook/kubernetes/). diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index f845cfe8..b3a7b921 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -1,24 +1,18 @@ apiVersion: v2 name: garage description: S3-compatible object store for small self-hosted geo-distributed deployments - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application +version: 0.7.3 +appVersion: "v1.3.1" +home: https://garagehq.deuxfleurs.fr/ +icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.0 +keywords: +- geo-distributed +- read-after-write-consistency +- s3-compatible -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "v0.7.2.1" +sources: +- https://git.deuxfleurs.fr/Deuxfleurs/garage.git + +maintainers: [] diff --git a/script/helm/garage/README.md b/script/helm/garage/README.md new file mode 100644 index 00000000..bdf69ec4 --- /dev/null +++ b/script/helm/garage/README.md @@ -0,0 +1,95 @@ +# garage + +![Version: 0.7.3](https://img.shields.io/badge/Version-0.7.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.1](https://img.shields.io/badge/AppVersion-v1.3.1-informational?style=flat-square) + +S3-compatible object store for small self-hosted geo-distributed deployments + +**Homepage:** + +## Source Code + +* + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet | +| deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) | +| deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start | +| environment | object | `{}` | | +| extraVolumeMounts | object | `{}` | | +| extraVolumes | object | `{}` | | +| fullnameOverride | string | `""` | | +| garage.blockSize | string | `"1048576"` | Defaults is 1MB An increase can result in better performance in certain scenarios https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size | +| garage.bootstrapPeers | list | `[]` | This is not required if you use the integrated kubernetes discovery | +| garage.compressionLevel | string | `"1"` | zstd compression level of stored blocks https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level | +| garage.dbEngine | string | `"lmdb"` | Can be changed for better performance on certain systems https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0 | +| garage.existingConfigMap | string | `""` | if not empty string, allow using an existing ConfigMap for the garage.toml, if set, ignores garage.toml | +| garage.garageTomlString | string | `""` | String Template for the garage configuration if set, ignores above values. Values can be templated, see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ | +| garage.kubernetesSkipCrd | bool | `false` | Set to true if you want to use k8s discovery but install the CRDs manually outside of the helm chart, for example if you operate at namespace level without cluster ressources | +| garage.metadataAutoSnapshotInterval | string | `""` | If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval | +| garage.replicationMode | string | `"3"` | Default to 3 replicas, see the replication_mode section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode | +| garage.rpcBindAddr | string | `"[::]:3901"` | | +| garage.rpcSecret | string | `""` | If not given, a random secret will be generated and stored in a Secret object | +| garage.s3.api.region | string | `"garage"` | | +| garage.s3.api.rootDomain | string | `".s3.garage.tld"` | | +| garage.s3.web.index | string | `"index.html"` | | +| garage.s3.web.rootDomain | string | `".web.garage.tld"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"dxflrs/amd64_garage"` | default to amd64 docker image | +| image.tag | string | `""` | set the image tag, please prefer using the chart version and not this to avoid compatibility issues | +| imagePullSecrets | list | `[]` | set if you need credentials to pull your custom image | +| ingress.s3.api.annotations | object | `{}` | Rely _either_ on the className or the annotation below but not both! If you want to use the className, set className: "nginx" and replace "nginx" by an Ingress controller name, examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). | +| ingress.s3.api.enabled | bool | `false` | | +| ingress.s3.api.hosts[0] | object | `{"host":"s3.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | garage S3 API endpoint, to be used with awscli for example | +| ingress.s3.api.hosts[1] | object | `{"host":"*.s3.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | garage S3 API endpoint, DNS style bucket access | +| ingress.s3.api.labels | object | `{}` | | +| ingress.s3.api.tls | list | `[]` | | +| ingress.s3.web.annotations | object | `{}` | Rely _either_ on the className or the annotation below but not both! If you want to use the className, set className: "nginx" and replace "nginx" by an Ingress controller name, examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). | +| ingress.s3.web.enabled | bool | `false` | | +| ingress.s3.web.hosts[0] | object | `{"host":"*.web.garage.tld","paths":[{"path":"/","pathType":"Prefix"}]}` | wildcard website access with bucket name prefix | +| ingress.s3.web.hosts[1] | object | `{"host":"mywebpage.example.com","paths":[{"path":"/","pathType":"Prefix"}]}` | specific bucket access with FQDN bucket | +| ingress.s3.web.labels | object | `{}` | | +| ingress.s3.web.tls | list | `[]` | | +| initImage.pullPolicy | string | `"IfNotPresent"` | | +| initImage.repository | string | `"busybox"` | | +| initImage.tag | string | `"stable"` | | +| livenessProbe | object | `{}` | Specifies a livenessProbe | +| monitoring.metrics.enabled | bool | `false` | If true, a service for monitoring is created with a prometheus.io/scrape annotation | +| monitoring.metrics.serviceMonitor.enabled | bool | `false` | If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator | +| monitoring.metrics.serviceMonitor.interval | string | `"15s"` | | +| monitoring.metrics.serviceMonitor.labels | object | `{}` | | +| monitoring.metrics.serviceMonitor.path | string | `"/metrics"` | | +| monitoring.metrics.serviceMonitor.relabelings | list | `[]` | | +| monitoring.metrics.serviceMonitor.scheme | string | `"http"` | | +| monitoring.metrics.serviceMonitor.scrapeTimeout | string | `"10s"` | | +| monitoring.metrics.serviceMonitor.tlsConfig | object | `{}` | | +| monitoring.tracing.sink | string | `""` | specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317` | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| persistence.data.hostPath | string | `"/var/lib/garage/data"` | | +| persistence.data.size | string | `"100Mi"` | | +| persistence.enabled | bool | `true` | | +| persistence.meta.hostPath | string | `"/var/lib/garage/meta"` | | +| persistence.meta.size | string | `"100Mi"` | | +| podAnnotations | object | `{}` | additonal pod annotations | +| podSecurityContext.fsGroup | int | `1000` | | +| podSecurityContext.runAsGroup | int | `1000` | | +| podSecurityContext.runAsNonRoot | bool | `true` | | +| podSecurityContext.runAsUser | int | `1000` | | +| readinessProbe | object | `{}` | Specifies a readinessProbe | +| resources | object | `{}` | | +| securityContext.capabilities | object | `{"drop":["ALL"]}` | The default security context is heavily restricted, feel free to tune it to your requirements | +| securityContext.readOnlyRootFilesystem | bool | `true` | | +| service.s3.api.port | int | `3900` | | +| service.s3.web.port | int | `3902` | | +| service.type | string | `"ClusterIP"` | You can rely on any service to expose your cluster - ClusterIP (+ Ingress) - NodePort (+ Ingress) - LoadBalancer | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml index 5cc7a45e..ab5b84db 100644 --- a/script/helm/garage/templates/configmap.yaml +++ b/script/helm/garage/templates/configmap.yaml @@ -1,7 +1,53 @@ +{{- if not .Values.garage.existingConfigMap }} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "garage.fullname" . }}-config data: garage.toml: |- - {{- tpl (index (index .Values.garage) "garage.toml") $ | nindent 4 }} + {{- if .Values.garage.garageTomlString }} + {{- tpl (index (index .Values.garage) "garageTomlString") $ | nindent 4 }} + {{- else }} + metadata_dir = "/mnt/meta" + data_dir = "/mnt/data" + + db_engine = "{{ .Values.garage.dbEngine }}" + + block_size = {{ .Values.garage.blockSize }} + + replication_mode = "{{ .Values.garage.replicationMode }}" + + compression_level = {{ .Values.garage.compressionLevel }} + + {{- if .Values.garage.metadataAutoSnapshotInterval }} + metadata_auto_snapshot_interval = {{ .Values.garage.metadataAutoSnapshotInterval | quote }} + {{- end }} + + rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" + # rpc_secret will be populated by the init container from a k8s secret object + rpc_secret = "__RPC_SECRET_REPLACE__" + + bootstrap_peers = {{ .Values.garage.bootstrapPeers }} + + [kubernetes_discovery] + namespace = "{{ .Release.Namespace }}" + service_name = "{{ include "garage.fullname" . }}" + skip_crd = {{ .Values.garage.kubernetesSkipCrd }} + + [s3_api] + s3_region = "{{ .Values.garage.s3.api.region }}" + api_bind_addr = "[::]:3900" + root_domain = "{{ .Values.garage.s3.api.rootDomain }}" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = "{{ .Values.garage.s3.web.rootDomain }}" + index = "{{ .Values.garage.s3.web.index }}" + + [admin] + api_bind_addr = "[::]:3903" + {{- if .Values.monitoring.tracing.sink }} + trace_sink = "{{ .Values.monitoring.tracing.sink }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/script/helm/garage/templates/service-headless.yaml b/script/helm/garage/templates/service-headless.yaml new file mode 100644 index 00000000..7bc9f2cc --- /dev/null +++ b/script/helm/garage/templates/service-headless.yaml @@ -0,0 +1,22 @@ +{{- if eq .Values.deployment.kind "StatefulSet" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "garage.fullname" . }}-headless + labels: + {{- include "garage.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ .Values.service.s3.api.port }} + targetPort: 3900 + protocol: TCP + name: s3-api + - port: {{ .Values.service.s3.web.port }} + targetPort: 3902 + protocol: TCP + name: s3-web + selector: + {{- include "garage.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/script/helm/garage/templates/service.yaml b/script/helm/garage/templates/service.yaml index 37218872..887c90d0 100644 --- a/script/helm/garage/templates/service.yaml +++ b/script/helm/garage/templates/service.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "garage.fullname" . }} labels: {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.service.type }} ports: @@ -37,4 +41,4 @@ spec: name: metrics selector: {{- include "garage.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/script/helm/garage/templates/workload.yaml b/script/helm/garage/templates/workload.yaml index 340c0054..d144cb41 100644 --- a/script/helm/garage/templates/workload.yaml +++ b/script/helm/garage/templates/workload.yaml @@ -10,11 +10,11 @@ spec: {{- include "garage.selectorLabels" . | nindent 6 }} {{- if eq .Values.deployment.kind "StatefulSet" }} replicas: {{ .Values.deployment.replicaCount }} - serviceName: {{ include "garage.fullname" . }} + serviceName: {{ include "garage.fullname" . }}-headless + podManagementPolicy: {{ .Values.deployment.podManagementPolicy }} {{- end }} template: metadata: - annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} @@ -63,6 +63,10 @@ spec: name: web-api - containerPort: 3903 name: admin + {{- with .Values.environment }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: meta mountPath: /mnt/meta @@ -71,15 +75,17 @@ spec: - name: etc mountPath: /etc/garage.toml subPath: garage.toml - # TODO - # livenessProbe: - # httpGet: - # path: / - # port: 3900 - # readinessProbe: - # httpGet: - # path: / - # port: 3900 + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} volumes: @@ -105,6 +111,9 @@ spec: - name: data emptyDir: {} {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index 3a1e41b9..5e419fe2 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -4,14 +4,34 @@ # Garage configuration. These values go to garage.toml garage: - # Default to 3 replicas, see the replication_mode section at - # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + # -- Can be changed for better performance on certain systems + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0 + dbEngine: "lmdb" + + # -- Defaults is 1MB + # An increase can result in better performance in certain scenarios + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size + blockSize: "1048576" + + # -- Default to 3 replicas, see the replication_mode section at + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode replicationMode: "3" + + # -- zstd compression level of stored blocks + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level + compressionLevel: "1" + + # -- If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval + metadataAutoSnapshotInterval: "" + rpcBindAddr: "[::]:3901" - # If not given, a random secret will be generated and stored in a Secret object + # -- If not given, a random secret will be generated and stored in a Secret object rpcSecret: "" - # This is not required if you use the integrated kubernetes discovery + # -- This is not required if you use the integrated kubernetes discovery bootstrapPeers: [] + # -- Set to true if you want to use k8s discovery but install the CRDs manually outside + # of the helm chart, for example if you operate at namespace level without cluster ressources kubernetesSkipCrd: false s3: api: @@ -20,41 +40,16 @@ garage: web: rootDomain: ".web.garage.tld" index: "index.html" - # Template for the garage configuration - # Values can be templated - # ref: https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ - garage.toml: |- - metadata_dir = "/mnt/meta" - data_dir = "/mnt/data" - replication_mode = "{{ .Values.garage.replicationMode }}" + # -- if not empty string, allow using an existing ConfigMap for the garage.toml, + # if set, ignores garage.toml + existingConfigMap: "" - rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" - # rpc_secret will be populated by the init container from a k8s secret object - rpc_secret = "__RPC_SECRET_REPLACE__" - - bootstrap_peers = {{ .Values.garage.bootstrapPeers }} - - [kubernetes_discovery] - namespace = "{{ .Release.Namespace }}" - service_name = "{{ include "garage.fullname" . }}" - skip_crd = {{ .Values.garage.kubernetesSkipCrd }} - - [s3_api] - s3_region = "{{ .Values.garage.s3.api.region }}" - api_bind_addr = "[::]:3900" - root_domain = "{{ .Values.garage.s3.api.rootDomain }}" - - [s3_web] - bind_addr = "[::]:3902" - root_domain = "{{ .Values.garage.s3.web.rootDomain }}" - index = "{{ .Values.garage.s3.web.index }}" - - [admin] - api_bind_addr = "[::]:3903" - {{- if .Values.monitoring.tracing.sink }} - trace_sink = "{{ .Values.monitoring.tracing.sink }}" - {{- end }} + # -- String Template for the garage configuration + # if set, ignores above values. + # Values can be templated, + # see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garageTomlString: "" # Data persistence persistence: @@ -72,14 +67,18 @@ persistence: # Deployment configuration deployment: - # Switchable to DaemonSet + # -- Switchable to DaemonSet kind: StatefulSet - # Number of StatefulSet replicas/garage nodes to start + # -- Number of StatefulSet replicas/garage nodes to start replicaCount: 3 + # -- If using statefulset, allow Parallel or OrderedReady (default) + podManagementPolicy: OrderedReady image: + # -- default to amd64 docker image repository: dxflrs/amd64_garage - # please prefer using the chart version and not this tag + # -- set the image tag, please prefer using the chart version and not this + # to avoid compatibility issues tag: "" pullPolicy: IfNotPresent @@ -88,19 +87,21 @@ initImage: tag: stable pullPolicy: IfNotPresent +# -- set if you need credentials to pull your custom image imagePullSecrets: [] nameOverride: "" fullnameOverride: "" serviceAccount: - # Specifies whether a service account should be created + # -- Specifies whether a service account should be created create: true - # Annotations to add to the service account + # -- Annotations to add to the service account annotations: {} - # The name of the service account to use. + # -- The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" +# -- additonal pod annotations podAnnotations: {} podSecurityContext: @@ -110,7 +111,7 @@ podSecurityContext: runAsNonRoot: true securityContext: - # The default security context is heavily restricted + # -- The default security context is heavily restricted, # feel free to tune it to your requirements capabilities: drop: @@ -118,11 +119,13 @@ securityContext: readOnlyRootFilesystem: true service: - # You can rely on any service to expose your cluster + # -- You can rely on any service to expose your cluster # - ClusterIP (+ Ingress) # - NodePort (+ Ingress) # - LoadBalancer type: ClusterIP + # -- Annotations to add to the service + annotations: {} s3: api: port: 3900 @@ -134,20 +137,23 @@ ingress: s3: api: enabled: false - # Rely either on the className or the annotation below but not both - # replace "nginx" by an Ingress controller - # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers + # -- Rely _either_ on the className or the annotation below but not both! + # If you want to use the className, set # className: "nginx" + # and replace "nginx" by an Ingress controller name, + # examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). annotations: {} # kubernetes.io/ingress.class: "nginx" # kubernetes.io/tls-acme: "true" labels: {} hosts: - - host: "s3.garage.tld" # garage S3 API endpoint + # -- garage S3 API endpoint, to be used with awscli for example + - host: "s3.garage.tld" paths: - path: / pathType: Prefix - - host: "*.s3.garage.tld" # garage S3 API endpoint, DNS style bucket access + # -- garage S3 API endpoint, DNS style bucket access + - host: "*.s3.garage.tld" paths: - path: / pathType: Prefix @@ -157,20 +163,23 @@ ingress: # - kubernetes.docker.internal web: enabled: false - # Rely either on the className or the annotation below but not both - # replace "nginx" by an Ingress controller - # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers + # -- Rely _either_ on the className or the annotation below but not both! + # If you want to use the className, set # className: "nginx" + # and replace "nginx" by an Ingress controller name, + # examples [here](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers). annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" labels: {} hosts: - - host: "*.web.garage.tld" # wildcard website access with bucket name prefix + # -- wildcard website access with bucket name prefix + - host: "*.web.garage.tld" paths: - path: / pathType: Prefix - - host: "mywebpage.example.com" # specific bucket access with FQDN bucket + # -- specific bucket access with FQDN bucket + - host: "mywebpage.example.com" paths: - path: / pathType: Prefix @@ -188,18 +197,39 @@ resources: {} # cpu: 100m # memory: 512Mi +# -- Specifies a livenessProbe +livenessProbe: {} + #httpGet: + # path: /health + # port: 3903 + #initialDelaySeconds: 5 + #periodSeconds: 30 +# -- Specifies a readinessProbe +readinessProbe: {} + #httpGet: + # path: /health + # port: 3903 + #initialDelaySeconds: 5 + #periodSeconds: 30 + nodeSelector: {} tolerations: [] affinity: {} +environment: {} + +extraVolumes: {} + +extraVolumeMounts: {} + monitoring: metrics: - # If true, a service for monitoring is created with a prometheus.io/scrape annotation + # -- If true, a service for monitoring is created with a prometheus.io/scrape annotation enabled: false serviceMonitor: - # If true, a ServiceMonitor CRD is created for a prometheus operator + # -- If true, a ServiceMonitor CRD is created for a prometheus operator # https://github.com/coreos/prometheus-operator enabled: false path: /metrics @@ -211,4 +241,5 @@ monitoring: scrapeTimeout: 10s relabelings: [] tracing: - sink: "" \ No newline at end of file + # -- specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317` + sink: "" diff --git a/script/jepsen.garage/.envrc b/script/jepsen.garage/.envrc new file mode 100644 index 00000000..1d953f4b --- /dev/null +++ b/script/jepsen.garage/.envrc @@ -0,0 +1 @@ +use nix diff --git a/script/jepsen.garage/.gitignore b/script/jepsen.garage/.gitignore new file mode 100644 index 00000000..31842a96 --- /dev/null +++ b/script/jepsen.garage/.gitignore @@ -0,0 +1,17 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +/.prepl-port +.hgignore +.hg/ +.direnv +/store +/store.* +.vagrant diff --git a/script/jepsen.garage/README.md b/script/jepsen.garage/README.md new file mode 100644 index 00000000..50c7eb38 --- /dev/null +++ b/script/jepsen.garage/README.md @@ -0,0 +1,166 @@ +# jepsen.garage + +Jepsen checking of Garage consistency properties. + +## Usage + +Requirements: + +- vagrant +- VirtualBox, configured so that nodes can take an IP in a private network `192.168.56.0/24` (it's the default) +- a user that can create VirtualBox VMs +- leiningen +- gnuplot + +Set up VMs before running tests: + +``` +vagrant up +``` + +Run tests: see commands below. + + +## Results + +### Register linear, without timestamp patch + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 20 --workload reg1 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: without the timestamp patch, nodes will create objects using their + local clock only as a timestamp, so the ordering will be all over the place if + clocks are scrambled. + +Results with timestamp patch (`--patch tsfix2`): + +- No failure with clock-scramble nemesis + +- Fails with clock-scramble nemesis + partition nemesis (`--scenario cp`). + +**This test is expected to fail.** +Indeed, S3 objects are not meant to behave like linearizable registers. +TODO explain using a counter-example + + +### Read-after-write CRDT register model + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload reg2 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: old values are not overwritten correctly when their timestamps are in the future. + +Results with timestamp patch (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + partition nemesis (`--scenario cp`). + This proves that `tsfix2` (PR#543) does improve consistency. + +- **Fails with layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage reg2/20231024T120806.899+0200`. + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +### Set, basic test (write some items, then read) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 200 --concurrency 200 --workload set1 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run + +- Does not seem to fail with only the layout reconfiguation nemesis (<10 runs), although theoretically it could + +- **Fails with the partition + layout reconfiguration nemesis** (`--scenario pr`). + Example of a failed run: `garage set1/20231024T172214.488+0200` (1 failure in 4 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- The tests are buggy and often result in an "unknown" validity status, which + is caused by some requests not returning results during network partitions or + other nemesis-induced broken cluster states. However, when the tests were + able to finish, there were no failures with scenarios `r`, `pr`, `cpr`, + `dpr`. + + +### Set, continuous test (interspersed reads and writes) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload set2 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + db nemesis + partition nemesis (`--scenario cdp`) (0 failures in 10 runs). + +- **Fails with just layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage set2/20231025T141940.198+0200` (10 failures in 10 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +## NLnet task 3 final results + +- With code from task3 (`--patch task3c`): [reg2 and set2](results/Results-2023-12-13-task3c.png), [set1](results/Results-2023-12-14-task3-set1.png). +- Without (`--patch tsfix2`): [reg2 and set2](results/Results-2023-12-13-tsfix2.png), set1 TBD. + +## Investigating (and fixing) errors + +### Segfaults + +They are due to the download being interrupted in the middle (^C during first launch on clean VMs), the `garage` binary is truncated. +Add `:force?` to the `cached-wget!` call in `daemon.clj` to re-download the binary, +or restar the VMs to clear temporary files. + +### In `jepsen.garage`: prefix wierdness + +In `store/garage set1/20231019T163358.615+0200`: + +``` +INFO [2023-10-19 16:35:20,977] clojure-agent-send-off-pool-207 - jepsen.garage.set list results for prefix set20/ : (set13/0 set13/1 set13/10 set13/11 set13/12 set13/13 set13/14 set13/15 set13/16 set13/17 set13/18 set13/19 set13/2 set13/20 set13/21 set13/22 set13/23 set13/24 set13/25 set13/26 set13/27 set13/28 set13/29 set13/3 set13/30 set13/31 set13/32 set13/33 set13/34 set13/35 set13/36 set13/37 set13/38 set13/39 set13/4 set13/40 set13/41 set13/42 set13/43 set13/44 set13/45 set13/46 set13/47 set13/48 set13/49 set13/5 set13/50 set13/51 set13/52 set13/53 set13/54 set13/55 set13/56 set13/57 set13/58 set13/59 set13/6 set13/60 set13/61 set13/62 set13/63 set13/64 set13/65 set13/66 set13/67 set13/68 set13/69 set13/7 set13/70 set13/71 set13/72 set13/73 set13/74 set13/75 set13/76 set13/77 set13/78 set13/79 set13/8 set13/80 set13/81 set13/82 set13/83 set13/84 set13/85 set13/86 set13/87 set13/88 set13/89 set13/9 set13/90 set13/91 set13/92 set13/93 set13/94 set13/95 set13/96 set13/97 set13/98 set13/99) (node: http://192.168.56.25:3900 ) +``` + +After inspecting, the actual S3 call made was with prefix "set13/", so at least this is not an error in Garage itself but in the jepsen code. + +Finally found out that this was due to closures not correctly capturing their context in the list function in s3api.clj (wtf clojure?) +Not sure exactly where it came from but it seems to have been fixed by making list-inner a separate function and not a sub-function, +and passing all values that were previously in the context (creds and prefix) as additional arguments. + +### `reg2` test inconsistency, even with timestamp fix + +The reg2 test is our custom checker for CRDT read-after-write on individual object keys, acting as registers which can be updated. +The test fails without the timestamp fix, which is expected as the clock scrambler will prevent nodes from having a correct ordering of objects. + +With the timestamp fix (`--patch tsfix1`), the happenned-before relationship should at least be respected, meaning that when a PutObject call starts +after another PutObject call has ended, the second call should overwrite the value of the first call, and that value should not be +readable by future GetObject calls. +However, we observed inconsistencies even with the timestamp fix. + +The inconsistencies seemed to always happenned after writing a nil value, which translates to a DeleteObject call +instead of a PutObject. By removing the possibility of writing nil values, therefore only doing +PutObject calls, the issue disappears. There is therefore an issue to fix in DeleteObject. + +The issue in DeleteObject seems to have been fixed by commit `c82d91c6bccf307186332b6c5c6fc0b128b1b2b1`, which can be used using `--patch tsfix2`. + + +## License + +Copyright © 2023 Alex Auvolat + +This program and the accompanying materials are made available under the +terms of the GNU Affero General Public License v3.0. diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile new file mode 100644 index 00000000..2ac24f98 --- /dev/null +++ b/script/jepsen.garage/Vagrantfile @@ -0,0 +1,40 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# + +def vm(config, hostname, ip) + config.vm.hostname = hostname + config.vm.network "private_network", ip: ip +end + +Vagrant.configure("2") do |config| + config.vm.box = "generic/debian10" + + config.vm.provider "virtualbox" do |vb| + vb.gui = false + vb.memory = "512" + vb.customize ["modifyvm", :id, "--vram=12"] + end + + config.vm.provision "shell", inline: <<-SHELL + echo "root:root" | chpasswd + mkdir -p /root/.ssh + echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys + SHELL + + config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end + config.vm.define "n2" do |config| vm(config, "n2", "192.168.56.22") end + config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end + config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end + config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end + config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end + config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end + + #config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end + #config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end + #config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end + #config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end + #config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end + #config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end + #config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end +end diff --git a/script/jepsen.garage/all_tests_1.sh b/script/jepsen.garage/all_tests_1.sh new file mode 100755 index 00000000..f9770865 --- /dev/null +++ b/script/jepsen.garage/all_tests_1.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c task3a tsfix2; do +for ppatch in v093 v1rc1; do + #for psc in c cp cdp r pr cpr dpr; do + for ptsk in reg2 set2; do + for psc in c cp cdp r pr cpr dpr; do + for irun in $(seq 10); do + lein run test --nodes-file nodes.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/all_tests_2.sh b/script/jepsen.garage/all_tests_2.sh new file mode 100755 index 00000000..641643ed --- /dev/null +++ b/script/jepsen.garage/all_tests_2.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c tsfix2; do +for ppatch in tsfix2; do + for psc in cdp r pr cpr dpr; do + for ptsk in set1; do + for irun in $(seq 10); do + lein run test --nodes-file nodes2.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/jaeger.sh b/script/jepsen.garage/jaeger.sh new file mode 100644 index 00000000..7f67b61b --- /dev/null +++ b/script/jepsen.garage/jaeger.sh @@ -0,0 +1,13 @@ +docker stop jaeger +docker rm jaeger + +# UI is on localhost:16686 +# otel-grpc collector is on localhost:4317 +# otel-http collector is on localhost:4318 + +docker run -d --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 4317:4317 \ + -p 4318:4318 \ + -p 16686:16686 \ + jaegertracing/all-in-one:1.50 diff --git a/script/jepsen.garage/nodes.vagrant b/script/jepsen.garage/nodes.vagrant new file mode 100644 index 00000000..9e5694e6 --- /dev/null +++ b/script/jepsen.garage/nodes.vagrant @@ -0,0 +1,7 @@ +192.168.56.21 +192.168.56.22 +192.168.56.23 +192.168.56.24 +192.168.56.25 +192.168.56.26 +192.168.56.27 diff --git a/script/jepsen.garage/nodes2.vagrant b/script/jepsen.garage/nodes2.vagrant new file mode 100644 index 00000000..842bf276 --- /dev/null +++ b/script/jepsen.garage/nodes2.vagrant @@ -0,0 +1,7 @@ +192.168.56.28 +192.168.56.29 +192.168.56.30 +192.168.56.31 +192.168.56.32 +192.168.56.33 +192.168.56.34 diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj new file mode 100644 index 00000000..59d45484 --- /dev/null +++ b/script/jepsen.garage/project.clj @@ -0,0 +1,10 @@ +(defproject jepsen.garage "0.1.0-SNAPSHOT" + :description "Jepsen testing for Garage" + :url "https://git.deuxfleurs.fr/Deuxfleurs/garage" + :license {:name "AGPLv3" + :url "https://www.gnu.org/licenses/agpl-3.0.en.html"} + :main jepsen.garage + :dependencies [[org.clojure/clojure "1.11.1"] + [jepsen "0.3.3-SNAPSHOT"] + [amazonica "0.3.163"]] + :repl-options {:init-ns jepsen.garage}) diff --git a/script/jepsen.garage/results/Results-2023-11-16.png b/script/jepsen.garage/results/Results-2023-11-16.png new file mode 100644 index 00000000..26dac833 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-11-16.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-13-task3c.png b/script/jepsen.garage/results/Results-2023-12-13-task3c.png new file mode 100644 index 00000000..216043c3 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-13-task3c.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png new file mode 100644 index 00000000..147d25e9 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png differ diff --git a/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png new file mode 100644 index 00000000..dbff3a95 Binary files /dev/null and b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png differ diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix new file mode 100644 index 00000000..01e4c845 --- /dev/null +++ b/script/jepsen.garage/shell.nix @@ -0,0 +1,18 @@ +{ pkgs ? import { + overlays = [ + (self: super: { + jdk = super.jdk11; + jre = super.jre11; + }) + ]; +} }: +pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + leiningen + jdk + jna + vagrant + gnuplot + graphviz + ]; +} diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj new file mode 100644 index 00000000..053ad8b6 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -0,0 +1,107 @@ +(ns jepsen.garage + (:require + [clojure.string :as str] + [jepsen + [checker :as checker] + [cli :as cli] + [generator :as gen] + [nemesis :as nemesis] + [tests :as tests]] + [jepsen.os.debian :as debian] + [jepsen.garage + [daemon :as grg] + [nemesis :as grgNemesis] + [reg :as reg] + [set :as set]])) + +(def workloads + "A map of workload names to functions that construct workloads, given opts." + {"reg1" reg/workload1 + "reg2" reg/workload2 + "set1" set/workload1 + "set2" set/workload2}) + +(def scenari + "A map of scenari to the associated nemesis" + {"c" grgNemesis/scenario-c + "cp" grgNemesis/scenario-cp + "r" grgNemesis/scenario-r + "pr" grgNemesis/scenario-pr + "cpr" grgNemesis/scenario-cpr + "cdp" grgNemesis/scenario-cdp + "dpr" grgNemesis/scenario-dpr}) + +(def patches + "A map of patch names to Garage builds" + {"default" "v0.9.0" + "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09" + "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1" + "task3a" "707442f5de416fdbed4681a33b739f0a787b7834" + "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997" + "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b" + "v093" "v0.9.3" + "v1rc1" "v1.0.0-rc1"}) + +(def cli-opts + "Additional command line options." + [["-p" "--patch NAME" "Garage patch to use" + :default "default" + :validate [patches (cli/one-of patches)]] + ["-s" "--scenario NAME" "Nemesis scenario to run" + :default "cp" + :validate [scenari (cli/one-of scenari)]] + ["-r" "--rate HZ" "Approximate number of requests per second, per thread." + :default 10 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--ops-per-key NUM" "Maximum number of operations on any given key." + :default 100 + :parse-fn parse-long + :validate [pos? "Must be a positive integer."]] + ["-w" "--workload NAME" "Workload of test to run" + :default "reg1" + :validate [workloads (cli/one-of workloads)]]]) + +(defn garage-test + "Given an options map from the command line runner (e.g. :nodes, :ssh, + :concurrency, ...), constructs a test map." + [opts] + (let [garage-version (get patches (:patch opts)) + db (grg/db garage-version) + workload ((get workloads (:workload opts)) opts) + scenario ((get scenari (:scenario opts)) (assoc opts :db db))] + (merge tests/noop-test + opts + {:pure-generators true + :name (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts))) + :os debian/os + :db db + :client (:client workload) + :generator (gen/phases + (->> + (:generator workload) + (gen/stagger (/ (:rate opts))) + (gen/nemesis (:generator scenario)) + (gen/time-limit (:time-limit opts))) + (gen/log "Healing cluster") + (gen/nemesis (:final-generator scenario)) + (gen/log "Waiting for recovery") + (gen/sleep 10) + (gen/log "Running final generator") + (gen/clients (:final-generator workload)) + (gen/log "Generators all done")) + :nemesis (:nemesis scenario) + :checker (checker/compose + {:perf (checker/perf (:perf scenario)) + :workload (:checker workload)}) + }))) + + +(defn -main + "Handles command line arguments. Can either run a test, or a web server for + browsing results." + [& args] + (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj new file mode 100644 index 00000000..0ea773fb --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -0,0 +1,152 @@ +(ns jepsen.garage.daemon + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [db :as db]] + [jepsen.control.util :as cu])) + +; CONSTANTS -- HOW GARAGE IS SET UP + +(def base-dir "/opt/garage") +(def data-dir (str base-dir "/data")) +(def meta-dir (str base-dir "/meta")) +(def binary (str base-dir "/garage")) +(def logfile (str base-dir "/garage.log")) +(def pidfile (str base-dir "/garage.pid")) + +(def admin-token "icanhazadmin") +(def access-key-id "GK8bfb6a51286071c6c9cd8bc3") +(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3") +(def bucket-name "jepsen") + +; THE GARAGE DB + +(defn install! + "Download and install Garage" + [node version] + (c/su + (c/trace + (info node "installing garage" version) + (c/exec :mkdir :-p base-dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/cached-wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary)))) + +(defn configure! + "Configure Garage" + [node] + (c/su + (c/trace + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" data-dir "\"\n" + "metadata_dir = \"" meta-dir "\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" admin-token "\"\n" + "trace_sink = \"http://192.168.56.1:4317\"\n") + "/etc/garage.toml")))) + +(defn connect-node! + "Connect a Garage node to the rest of the cluster" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id))))) + +(defn configure-node! + "Configure a Garage node to be part of a cluster layout" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (c/on (jepsen/primary test) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) + +(defn finalize-config! + "Apply the layout and create a key/bucket pair in the cluster" + [node] + (c/trace + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :import access-key-id secret-access-key :--yes) + (c/exec binary :bucket :create bucket-name) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id) + (info node "key info: " (c/exec binary :key :info access-key-id)))) + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (install! node version) + (configure! node) + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server) + (c/exec :sleep 3) + + (jepsen/synchronize test) + (connect-node! test node) + + (jepsen/synchronize test) + (configure-node! test node) + + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (finalize-config! node))) + + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf logfile) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf meta-dir))) + + db/Pause + (pause! [_ test node] + (cu/grepkill! :stop binary)) + (resume! [_ test node] + (cu/grepkill! :cont binary)) + + db/Kill + (kill! [_ test node] + (cu/stop-daemon! binary pidfile)) + (start! [_ test node] + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server)) + + db/LogFiles + (log-files [_ test node] + [logfile]))) + +(defn creds + "Obtain Garage credentials for node" + [node] + {:access-key access-key-id + :secret-key secret-access-key + :endpoint (str "http://" node ":3900") + :bucket bucket-name + :client-config {:path-style-access-enabled true}}) + diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj new file mode 100644 index 00000000..dfce0255 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -0,0 +1,142 @@ +(ns jepsen.garage.nemesis + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [generator :as gen] + [nemesis :as nemesis]] + [jepsen.nemesis.combined :as combined] + [jepsen.garage.daemon :as grg] + [jepsen.control.util :as cu])) + +; ---- reconfiguration nemesis ---- + +(defn configure-present! + "Configure node to be active in new cluster layout" + [test nodes] + (info "configure-present!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs))))) + +(defn configure-absent! + "Configure nodes to be active in new cluster layout" + [test nodes] + (info "configure-absent!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs))))) + +(defn finalize-config! + "Apply the proposed cluster layout" + [test] + (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) + [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] + (if layout-next-version + (do + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version))) + (info "no layout changes to apply")))) + +(defn reconfigure-subset + "Reconfigure cluster with only a subset of nodes" + [cnt] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] op + (case (:f op) + :start + (let [[keep-nodes remove-nodes] + (->> (:nodes test) + shuffle + (split-at cnt))] + (info "layout split: keep " keep-nodes ", remove " remove-nodes) + (configure-present! test keep-nodes) + (configure-absent! test remove-nodes) + (finalize-config! test) + (assoc op :value keep-nodes)) + :stop + (do + (info "layout un-split: all nodes=" (:nodes test)) + (configure-present! test (:nodes test)) + (finalize-config! test) + (assoc op :value (:nodes test))))) + + (teardown! [this test] this))) + +; ---- nemesis scenari ---- + +(defn nemesis-op + "A generator for a single nemesis operation" + [op] + (fn [_ _] {:type :info, :f op})) + +(defn reconfiguration-package + "Cluster reconfiguration nemesis package" + [opts] + {:generator (->> + (gen/mix [(nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger (:interval opts 5))) + :final-generator {:type :info, :f :reconfigure-stop} + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)}) + :perf #{{:name "reconfigure" + :start #{:reconfigure-start} + :stop #{:reconfigur-stop} + :color "#A197E9"}}}) + +(defn scenario-c + "Clock modifying scenario" + [opts] + (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})) + +(defn scenario-cp + "Clock modifying + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-r + "Cluster reconfiguration scenario" + [opts] + (reconfiguration-package {:interval 1})) + +(defn scenario-pr + "Partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cpr + "Clock scramble + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cdp + "Clock modifying + db + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-dpr + "Db + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj new file mode 100644 index 00000000..39708c0b --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -0,0 +1,143 @@ +(ns jepsen.garage.reg + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-get [_ _] {:type :invoke, :f :read, :value nil}) +(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))}) +(defn op-del [_ _] {:type :invoke, :f :write, :value nil}) + +(defrecord RegClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op)] + (case (:f op) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (let [value (s3/get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value)))) + :write + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) k v) + (assoc op :type :ok))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn reg-read-after-write + "Read-after-Write checker for register operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:put-values {-1 nil} + :put-done #{-1} + :put-in-progress {} + :read-can-contain {} + :bad-reads #{}} + final (reduce + (fn [state op] + (let [current-values (set/union + (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state))) + (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state)))) + read-can-contain (reduce + (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v))) + {} (:read-can-contain state))] + (info "--------") + (info "state: " state) + (info "current-values: " current-values) + (info "read-can-contain: " read-can-contain) + (info "op: " op) + (case [(:type op) (:f op)] + ([:invoke :write]) + (assoc state + :read-can-contain read-can-contain + :put-values (assoc (:put-values state) (:index op) (:value op)) + :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)])) + ([:ok :write]) + (let [[index overwrites] (get (:put-in-progress state) (:process op))] + (assoc state + :read-can-contain read-can-contain + :put-in-progress (dissoc (:put-in-progress state) (:process op)) + :put-done + (conj + (set/difference (:put-done state) overwrites) + index))) + ([:invoke :read]) + (assoc state + :read-can-contain (assoc read-can-contain (:process op) current-values)) + ([:ok :read]) + (let [this-read-can-contain (get read-can-contain (:process op)) + bad-reads (if (contains? this-read-can-contain (:value op)) + (:bad-reads state) + (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))] + (info "this-read-can-contain: " this-read-can-contain) + (assoc state + :read-can-contain (dissoc read-can-contain (:process op)) + :bad-reads bad-reads)) + state))) + init history) + valid? (empty? (:bad-reads final))] + (assoc final :valid? valid?))))) + +(defn workload-common + "Common parts of workload" + [opts] + {:client (RegClient. nil) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> + (gen/mix [op-get op-put op-del]) + (gen/limit (:ops-per-key opts)))))}) + +(defn workload1 + "Tests linearizable reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable + {:model (model/register) + :algorithm :linear}) + :timeline (timeline/html)})))) + +(defn workload2 + "Tests CRDT reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:reg-read-after-write (reg-read-after-write) + :timeline (timeline/html)})))) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj new file mode 100644 index 00000000..531e0157 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -0,0 +1,48 @@ +(ns jepsen.garage.s3api + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c]] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; GARAGE S3 HELPER FUNCTIONS + +(defn get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn list-inner [creds prefix ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner creds prefix (:next-continuation-token list-result) objects) + objects))) +(defn list + "Helper for ListObjects -- just lists everything in the bucket" + [creds prefix] + (list-inner creds prefix nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj new file mode 100644 index 00000000..2c7a2ccd --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -0,0 +1,135 @@ +(ns jepsen.garage.set + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [checker :as checker] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)}) +(defn op-read [_ _] {:type :invoke, :f :read, :value nil}) + +(defrecord SetClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op) + prefix (str "set" k "/")] + (case (:f op) + :add + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) (str prefix v) "present") + (assoc op :type :ok))) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (do + (let [items (s3/list (:creds this) prefix)] + (let [items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn set-read-after-write + "Read-after-Write checker for set operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:add-started #{} + :add-done #{} + :read-must-contain {} + :missed #{} + :unexpected #{}} + final (reduce + (fn [state op] + (case [(:type op) (:f op)] + ([:invoke :add]) + (assoc state :add-started (conj (:add-started state) (:value op))) + ([:ok :add]) + (assoc state :add-done (conj (:add-done state) (:value op))) + ([:invoke :read]) + (assoc-in state [:read-must-contain (:process op)] (:add-done state)) + ([:ok :read]) + (let [read-must-contain (get (:read-must-contain state) (:process op)) + new-missed (set/difference read-must-contain (:value op)) + new-unexpected (set/difference (:value op) (:add-started state))] + (assoc state + :read-must-contain (dissoc (:read-must-contain state) (:process op)) + :missed (set/union (:missed state) new-missed), + :unexpected (set/union (:unexpected state) new-unexpected))) + state)) + init history) + valid? (and (empty? (:missed final)) (empty? (:unexpected final)))] + (assoc final :valid? valid?))))) + +(defn workload1 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set (checker/set) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x})) + (gen/limit (:ops-per-key opts))))) + :final-generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (gen/phases + (gen/once op-read) + (gen/sleep 5))))}) + +(defn workload2 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set-read-after-write (set-read-after-write) + ; :set-full (checker/set-full {:linearizable? false}) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> (gen/mix [op-add-rand100 op-read]) + (gen/limit (:ops-per-key opts)))))}) + + diff --git a/script/jepsen.garage/test/jepsen/garage_test.clj b/script/jepsen.garage/test/jepsen/garage_test.clj new file mode 100644 index 00000000..055392a1 --- /dev/null +++ b/script/jepsen.garage/test/jepsen/garage_test.clj @@ -0,0 +1,7 @@ +(ns jepsen.garage-test + (:require [clojure.test :refer :all] + [jepsen.garage :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1)))) diff --git a/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml b/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml new file mode 100644 index 00000000..cd0fb166 --- /dev/null +++ b/script/k8s/crd/garagenodes.deuxfleurs.fr.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: garagenodes.deuxfleurs.fr +spec: + conversion: + strategy: None + group: deuxfleurs.fr + names: + kind: GarageNode + listKind: GarageNodeList + plural: garagenodes + singular: garagenode + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for Node via `CustomResource` + properties: + spec: + properties: + address: + format: ip + type: string + hostname: + type: string + port: + format: uint16 + minimum: 0 + type: integer + required: + - address + - hostname + - port + type: object + required: + - spec + title: GarageNode + type: object + served: true + storage: true + subresources: {} \ No newline at end of file diff --git a/script/k8s/crd/kustomization.yaml b/script/k8s/crd/kustomization.yaml new file mode 100644 index 00000000..9f20eccf --- /dev/null +++ b/script/k8s/crd/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- garagenodes.deuxfleurs.fr.yaml \ No newline at end of file diff --git a/script/not-dynamic.sh b/script/not-dynamic.sh index b9a13070..2beb4e0b 100755 --- a/script/not-dynamic.sh +++ b/script/not-dynamic.sh @@ -7,7 +7,12 @@ if [ "$#" -ne 1 ]; then exit 2 fi -if file $1 | grep 'dynamically linked' 2>&1; then +if [ ! -x "$1" ]; then + echo "[fail] $1 does not exist or is not an executable" + exit 1 +fi + +if file "$1" | grep 'dynamically linked' 2>&1; then echo "[fail] $1 is dynamic" exit 1 fi diff --git a/script/test-renumbering.sh b/script/test-renumbering.sh new file mode 100644 index 00000000..0b1bd320 --- /dev/null +++ b/script/test-renumbering.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +: ' + This script tests part renumbering on an S3 remote (here configured for Minio). + + On Minio: + + The results confirm that if I upload parts with number 1, 4, 5 and 6, + they are renumbered to 1, 2, 3 and 4 after CompleteMultipartUpload. + Thus, specifying partNumber=4 on a GetObject/HeadObject should return + information on the part I originally uploaded with part number + + On S3: not tested + + Sample output (on Minio): + + f07e1404cc527d494242824ded3a616b part1 + 78974cd4d0f622eb3426ea7cd22f5a1c part4 + f9cc379f8baa61645558d9ba7e6351fa part5 + 1bd2383eebbac1f8e7143575ba5b1f4a part6 + Upload ID: 6838b813-d0ca-400b-9d28-ec8b2b5cd004 + PART 1 ETag: "f07e1404cc527d494242824ded3a616b" + PART 4 ETag: "78974cd4d0f622eb3426ea7cd22f5a1c" + PART 5 ETag: "f9cc379f8baa61645558d9ba7e6351fa" + PART 6 ETag: "1bd2383eebbac1f8e7143575ba5b1f4a" + ======================================== LIST ==== + { + "Parts": [ + { + "PartNumber": 1, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"f07e1404cc527d494242824ded3a616b\"", + "Size": 20971520 + }, + { + "PartNumber": 4, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"78974cd4d0f622eb3426ea7cd22f5a1c\"", + "Size": 20971520 + }, + { + "PartNumber": 5, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"f9cc379f8baa61645558d9ba7e6351fa\"", + "Size": 20971520 + }, + { + "PartNumber": 6, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"1bd2383eebbac1f8e7143575ba5b1f4a\"", + "Size": 20971520 + } + ], + "ChecksumAlgorithm": "", + "Initiator": { + "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" + }, + "Owner": { + "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" + }, + "StorageClass": "STANDARD" + } + ======================================== COMPLETE ==== + { + "Location": "http://localhost:9000/test/upload", + "Bucket": "test", + "Key": "upload", + "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"" + } + ======================================== LIST ==== + + An error occurred (NoSuchUpload) when calling the ListParts operation: The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. + ======================================== GET PART 4 ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:21:59+00:00", + "ContentLength": 20971520, + "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"", + "ContentRange": "bytes 62914560-83886079/83886080", + "ContentType": "binary/octet-stream", + "Metadata": {}, + "PartsCount": 4 + } + 1bd2383eebbac1f8e7143575ba5b1f4a get-part4 + + + Conclusions: + + - Parts are indeed renumbered with consecutive numbers + - ListParts only applies to multipart uploads in progress, + it cannot be used once the multipart upload has been completed +' + +export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU +export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV + +function aws { command aws --endpoint-url http://localhost:9000 $@ ; } + +aws --version + +aws s3 mb s3://test + +for NUM in 1 4 5 6; do + dd if=/dev/urandom of=part$NUM bs=1M count=10 +done +md5sum part* + +UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId") +echo "Upload ID: $UPLOAD" + +PARTS="" + +for NUM in 1 4 5 6; do + ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \ + --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag") + echo "PART $NUM ETag: $ETAG" + if [ -n "$PARTS" ]; then + PARTS="$PARTS," + fi + PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}" +done + +echo "======================================== LIST ====" +aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq + +echo "======================================== COMPLETE ====" +echo "{\"Parts\":[$PARTS]}" > mpu +aws s3api complete-multipart-upload --multipart-upload file://mpu \ + --bucket test --key 'upload' --upload-id "$UPLOAD" + +echo "======================================== LIST ====" +aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq + +echo "======================================== GET PART 4 ====" +aws s3api get-object --bucket test --key upload --part-number 4 get-part4 +md5sum get-part4 diff --git a/script/test-skip-part.sh b/script/test-skip-part.sh new file mode 100644 index 00000000..20ae017d --- /dev/null +++ b/script/test-skip-part.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +: ' + This script tests whether uploaded parts can be skipped in a + CompleteMultipartUpoad + + On Minio: yes, parts can be skipped + + On S3: not tested + + Sample output (on Minio): + + f23911bcd1230f5ebe8887cbf5bc396e part1 + a2657143167eaf647c40473e78a091dc part4 + 72f72c02c5163bc81024b28ac818c5e0 part5 + e29cf500d20498218904b8df8806caa2 part6 + Upload ID: e8fe7b83-9800-46fb-ae90-9d7ccd42fe76 + PART 1 ETag: "f23911bcd1230f5ebe8887cbf5bc396e" + PART 4 ETag: "a2657143167eaf647c40473e78a091dc" + PART 5 ETag: "72f72c02c5163bc81024b28ac818c5e0" + PART 6 ETag: "e29cf500d20498218904b8df8806caa2" + ======================================== COMPLETE ==== + { + "Location": "http://localhost:9000/test/upload", + "Bucket": "test", + "Key": "upload", + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"" + } + ======================================== GET FULL ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:54:35+00:00", + "ContentLength": 31457280, + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"", + "ContentType": "binary/octet-stream", + "Metadata": {} + } + 97fb904da7ad310699a6afab0eb6e061 get-full + 97fb904da7ad310699a6afab0eb6e061 - + ======================================== GET PART 3 ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:54:35+00:00", + "ContentLength": 10485760, + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"", + "ContentRange": "bytes 20971520-31457279/31457280", + "ContentType": "binary/octet-stream", + "Metadata": {}, + "PartsCount": 3 + } + e29cf500d20498218904b8df8806caa2 get-part3 + + Conclusions: + + - Skipping a part in a CompleteMultipartUpoad call is OK + - The part is simply not included in the stored object + - Sequential part renumbering counts only non-skipped parts +' + +export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU +export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV + +function aws { command aws --endpoint-url http://localhost:9000 $@ ; } + +aws --version + +aws s3 mb s3://test + +for NUM in 1 4 5 6; do + dd if=/dev/urandom of=part$NUM bs=1M count=10 +done +md5sum part* + +UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId") +echo "Upload ID: $UPLOAD" + +PARTS="" + +for NUM in 1 4 5 6; do + ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \ + --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag") + echo "PART $NUM ETag: $ETAG" + if [ "$NUM" != "5" ]; then + if [ -n "$PARTS" ]; then + PARTS="$PARTS," + fi + PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}" + fi +done + +echo "======================================== COMPLETE ====" +echo "{\"Parts\":[$PARTS]}" > mpu +aws s3api complete-multipart-upload --multipart-upload file://mpu \ + --bucket test --key 'upload' --upload-id "$UPLOAD" + +echo "======================================== GET FULL ====" +aws s3api get-object --bucket test --key upload get-full +md5sum get-full +cat part1 part4 part6 | md5sum + +echo "======================================== GET PART 3 ====" +aws s3api get-object --bucket test --key upload --part-number 3 get-part3 +md5sum get-part3 diff --git a/script/test-smoke.sh b/script/test-smoke.sh index eababf38..acf56a90 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -31,6 +31,11 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB +dd if=/dev/urandom of=/tmp/garage.part1.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part2.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part3.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part4.rnd bs=1M count=5 + # data of lower entropy, to test compression dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64 dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64 @@ -40,7 +45,7 @@ echo "🧪 S3 API testing..." # AWS if [ -z "$SKIP_AWS" ]; then - echo "🛠️ Testing with awscli" + echo "🛠️ Testing with awscli (aws s3)" source ${SCRIPT_FOLDER}/dev-env-aws.sh aws s3 ls for idx in {1..3}.{rnd,b64}; do @@ -51,6 +56,45 @@ if [ -z "$SKIP_AWS" ]; then rm /tmp/garage.$idx.dl aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws" done + + echo "🛠️ Testing multipart uploads with awscli (aws s3api)" + UPLOAD=$(aws s3api create-multipart-upload --bucket eprouvette --key 'upload' | jq -r ".UploadId") + echo "Upload ID: $UPLOAD" + ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 3 --body "/tmp/garage.part1.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG2=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 2 --body "/tmp/garage.part2.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 3 --body "/tmp/garage.part3.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG6=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 6 --body "/tmp/garage.part4.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + MPU="{\"Parts\":[{\"PartNumber\":2,\"ETag\":$ETAG2}, {\"PartNumber\":3,\"ETag\":$ETAG3}, {\"PartNumber\":6,\"ETag\":$ETAG6}]}" + echo $MPU > /tmp/garage.mpu.json + aws s3api complete-multipart-upload --multipart-upload file:///tmp/garage.mpu.json \ + --bucket eprouvette --key 'upload' --upload-id "$UPLOAD" + aws s3api get-object --bucket eprouvette --key upload /tmp/garage.mpu.get + if [ "$(md5sum /tmp/garage.mpu.get | cut -d ' ' -f 1)" != "$(cat /tmp/garage.part{2,3,4}.rnd | md5sum | cut -d ' ' -f 1)" ]; then + echo "Invalid multipart upload" + exit 1 + fi + aws s3api delete-object --bucket eprouvette --key upload + + echo "🛠️ Test SSE-C with awscli (aws s3)" + SSEC_KEY="u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y=" + SSEC_KEY_MD5="jMGbs3GyZkYjJUP6q5jA7g==" + echo "$SSEC_KEY" | base64 -d > /tmp/garage.ssec-key + for idx in {1,2}.rnd; do + aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \ + "/tmp/garage.$idx" "s3://eprouvette/garage.$idx.aws.sse-c" + aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \ + "s3://eprouvette/garage.$idx.aws.sse-c" "/tmp/garage.$idx.dl.sse-c" + diff "/tmp/garage.$idx" "/tmp/garage.$idx.dl.sse-c" + aws s3api delete-object --bucket eprouvette --key "garage.$idx.aws.sse-c" + done fi # S3CMD @@ -141,6 +185,7 @@ rm eprouvette/winscp EOF fi +rm /tmp/garage.part{1..4}.rnd rm /tmp/garage.{1..3}.{rnd,b64} echo "🏁 Teardown" diff --git a/script/test-upgrade.sh b/script/test-upgrade.sh new file mode 100755 index 00000000..dc25e7c6 --- /dev/null +++ b/script/test-upgrade.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -ex + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +SCRIPT_FOLDER="`dirname \"$0\"`" +REPO_FOLDER="${SCRIPT_FOLDER}/../" +GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" +GARAGE_RELEASE="${REPO_FOLDER}/target/release/" +NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/" +PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" + +OLD_VERSION="$1" +ARCH="$2" + + +echo "Downloading old garage binary..." +curl https://garagehq.deuxfleurs.fr/_releases/$OLD_VERSION/$ARCH/garage > /tmp/old_garage +chmod +x /tmp/old_garage + +echo "============= insert data into old version cluster =================" + +export GARAGE_BIN=/tmp/old_garage +if echo $OLD_VERSION | grep 'v0\.8\.'; then + echo "Detected Garage v0.8.x" + export GARAGE_08=1 +fi + +echo "⏳ Setup cluster using old version" +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-clean.sh +${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 & +sleep 6 +${SCRIPT_FOLDER}/dev-configure.sh +${SCRIPT_FOLDER}/dev-bucket.sh + +echo "🛠️ Inserting data in old cluster" +source ${SCRIPT_FOLDER}/dev-env-rclone.sh +rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line + +echo "🏁 Stopping old cluster" +killall -INT old_garage +sleep 2 +killall -9 old_garage || true + +echo "🏁 Removing old garage version" +rm -rv $GARAGE_BIN +export -n GARAGE_BIN +export -n GARAGE_08 + +echo "================ read data from new cluster ===================" + +echo "⏳ Setup cluster using new version" +pwd +ls +export GARAGE_BIN=$(which garage) +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 & +sleep 3 + +echo "🛠️ Retrieving data from old cluster" +rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list + +if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then + echo "TEST FAILURE: directories are different" + exit 1 +fi +rm -r /tmp/test_dotgit + +echo "🏁 Teardown" +rm -rf /tmp/garage-{data,meta}-* +rm -rf /tmp/config.*.toml + +echo "✅ Success" diff --git a/shell.nix b/shell.nix index 0d510e33..c3dedca8 100644 --- a/shell.nix +++ b/shell.nix @@ -3,101 +3,53 @@ with import ./nix/common.nix; let - pkgs = import pkgsSrc { + pkgs = import nixpkgs { inherit system; - overlays = [ cargo2nixOverlay ]; }; - kaniko = (import ./nix/kaniko.nix) pkgs; - manifest-tool = (import ./nix/manifest-tool.nix) pkgs; winscp = (import ./nix/winscp.nix) pkgs; +in +{ + # --- Dev shell inherited from flake.nix --- + devShell = devShells.default; + devShellFull = devShells.full; -in { - # --- Rust Shell --- - # Use it to compile Garage - rust = pkgs.mkShell { - nativeBuildInputs = [ - #pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - #pkgs.clippy - pkgs.rustfmt - #pkgs.perl - #pkgs.protobuf - #pkgs.pkg-config - #pkgs.openssl - pkgs.file - #cargo2nix.packages.x86_64-linux.cargo2nix - ]; - }; - - # --- Integration shell --- - # Use it to test Garage with common S3 clients - integration = pkgs.mkShell { - nativeBuildInputs = [ + # --- Continuous integration shell --- + # The shell used for all CI jobs (along with devShell) + ci = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ winscp - pkgs.s3cmd - pkgs.awscli2 - pkgs.minio-client - pkgs.rclone - pkgs.socat - pkgs.psmisc - pkgs.which - pkgs.openssl - pkgs.curl - pkgs.jq + + kaniko + manifest-tool + awscli2 + file + s3cmd + minio-client + rclone + socat + psmisc + which + openssl + curl + jq ]; - }; - - # --- Release shell --- - # A shell built to make releasing easier - release = pkgs.mkShell { shellHook = '' - function refresh_toolchain { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec - } - - function refresh_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do - echo "Updating cache for ''${attr}" - derivation=$(nix-instantiate --attr ''${attr}) - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation%\!bin}) - done - rm /tmp/nix-signing-key.sec - } - - function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default devShell.x86_64-linux; do - echo "Updating cache for ''${attr}" - derivation=$(nix path-info --derivation ".#''${attr}") - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation}) - done - rm /tmp/nix-signing-key.sec - } + export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' function to_s3 { aws \ --endpoint-url https://garage.deuxfleurs.fr \ --region garage \ s3 cp \ - ./result-bin/bin/garage \ - s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage + ./result/bin/garage \ + s3://garagehq.deuxfleurs.fr/_releases/''${CI_COMMIT_TAG:-$CI_COMMIT_SHA}/''${TARGET}/garage } function to_docker { executor \ --force \ - --customPlatform="''${DOCKER_PLATFORM}" \ - --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ + --customPlatform="$(echo "''${DOCKER_PLATFORM}" | sed 's/i386/386/')" \ + --destination "$(echo "''${CONTAINER_NAME}" | sed 's/i386/386/'):''${CONTAINER_TAG}" \ --context dir://`pwd` \ --verbosity=debug } @@ -156,7 +108,25 @@ in { s3://garagehq.deuxfleurs.fr/ } ''; - nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ]; + + }; + + # --- Cache shell --- + # A shell for refreshing caches + cache = pkgs.mkShell { + shellHook = '' + function refresh_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in pkgs.amd64.debug test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do + echo "Updating cache for ''${attr}" + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') + + done + rm /tmp/nix-signing-key.sec + } + ''; }; } diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml deleted file mode 100644 index 40793fd5..00000000 --- a/src/api/Cargo.toml +++ /dev/null @@ -1,63 +0,0 @@ -[package] -name = "garage_api" -version = "0.8.2" -authors = ["Alex Auvolat "] -edition = "2018" -license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" -repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" -readme = "../../README.md" - -[lib] -path = "lib.rs" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -garage_model = { version = "0.8.2", path = "../model" } -garage_table = { version = "0.8.2", path = "../table" } -garage_block = { version = "0.8.2", path = "../block" } -garage_util = { version = "0.8.2", path = "../util" } -garage_rpc = { version = "0.8.2", path = "../rpc" } - -async-trait = "0.1.7" -base64 = "0.21" -bytes = "1.0" -chrono = "0.4" -crypto-common = "0.1" -err-derive = "0.3" -hex = "0.4" -hmac = "0.12" -idna = "0.3" -tracing = "0.1" -md-5 = "0.10" -nom = "7.1" -sha2 = "0.10" - -futures = "0.3" -futures-util = "0.3" -pin-project = "1.0.12" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-stream = "0.1" - -form_urlencoded = "1.0.0" -http = "0.2" -httpdate = "1.0" -http-range = "0.1" -hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } -multer = "2.0" -percent-encoding = "2.1.0" -roxmltree = "0.14" -serde = { version = "1.0", features = ["derive"] } -serde_bytes = "0.11" -serde_json = "1.0" -quick-xml = { version = "0.21", features = [ "serialize" ] } -url = "2.3" - -opentelemetry = "0.17" -opentelemetry-prometheus = { version = "0.10", optional = true } -prometheus = { version = "0.13", optional = true } - -[features] -k2v = [ "garage_util/k2v", "garage_model/k2v" ] -metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml new file mode 100644 index 00000000..656c6825 --- /dev/null +++ b/src/api/admin/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "garage_api_admin" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Admin API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +argon2.workspace = true +async-trait.workspace = true +thiserror.workspace = true +hex.workspace = true +tracing.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } + +[features] +metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 58dd38d8..6f0c474f 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -1,12 +1,11 @@ use std::collections::HashMap; -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; +use argon2::password_hash::PasswordHash; -use futures::future::Future; use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW}; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; +use tokio::sync::watch; use opentelemetry::trace::SpanRef; @@ -18,14 +17,19 @@ use prometheus::{Encoder, TextEncoder}; use garage_model::garage::Garage; use garage_rpc::system::ClusterHealthStatus; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; -use crate::generic_server::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; -use crate::admin::bucket::*; -use crate::admin::cluster::*; -use crate::admin::error::*; -use crate::admin::key::*; -use crate::admin::router::{Authorization, Endpoint}; +use crate::bucket::*; +use crate::cluster::*; +use crate::error::*; +use crate::key::*; +use crate::router_v0; +use crate::router_v1::{Authorization, Endpoint}; + +pub type ResBody = BoxBody; pub struct AdminApiServer { garage: Arc, @@ -41,14 +45,8 @@ impl AdminApiServer { #[cfg(feature = "metrics")] exporter: PrometheusExporter, ) -> Self { let cfg = &garage.config.admin; - let metrics_token = cfg - .metrics_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); - let admin_token = cfg - .admin_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); + let metrics_token = cfg.metrics_token.as_deref().map(hash_bearer_token); + let admin_token = cfg.admin_token.as_deref().map(hash_bearer_token); Self { garage, #[cfg(feature = "metrics")] @@ -60,28 +58,28 @@ impl AdminApiServer { pub async fn run( self, - bind_addr: SocketAddr, - shutdown_signal: impl Future, + bind_addr: UnixOrTCPSocketAddress, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { let region = self.garage.config.s3_api.s3_region.clone(); ApiServer::new(region, self) - .run_server(bind_addr, shutdown_signal) + .run_server(bind_addr, Some(0o220), must_exit) .await } - fn handle_options(&self, _req: &Request) -> Result, Error> { + fn handle_options(&self, _req: &Request) -> Result, Error> { Ok(Response::builder() .status(StatusCode::NO_CONTENT) .header(ALLOW, "OPTIONS, GET, POST") .header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS, GET, POST") .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .body(Body::empty())?) + .body(empty_body())?) } - async fn handle_check_website_enabled( + async fn handle_check_domain( &self, - req: Request, - ) -> Result, Error> { + req: Request, + ) -> Result, Error> { let query_params: HashMap = req .uri() .query() @@ -102,12 +100,56 @@ impl AdminApiServer { .get("domain") .ok_or_internal_error("Could not parse domain query string")?; - let bucket_id = self + if self.check_domain(domain).await? { + Ok(Response::builder() + .status(StatusCode::OK) + .body(string_body(format!( + "Domain '{domain}' is managed by Garage" + )))?) + } else { + Err(Error::bad_request(format!( + "Domain '{domain}' is not managed by Garage" + ))) + } + } + + async fn check_domain(&self, domain: &str) -> Result { + // Resolve bucket from domain name, inferring if the website must be activated for the + // domain to be valid. + let (bucket_name, must_check_website) = if let Some(bname) = self + .garage + .config + .s3_api + .root_domain + .as_ref() + .and_then(|rd| host_to_bucket(domain, rd)) + { + (bname.to_string(), false) + } else if let Some(bname) = self + .garage + .config + .s3_web + .as_ref() + .and_then(|sw| host_to_bucket(domain, sw.root_domain.as_str())) + { + (bname.to_string(), true) + } else { + (domain.to_string(), true) + }; + + let bucket_id = match self .garage .bucket_helper() - .resolve_global_bucket_name(&domain) + .resolve_global_bucket_name(&bucket_name) .await? - .ok_or(HelperError::NoSuchBucket(domain.to_string()))?; + { + Some(bucket_id) => bucket_id, + None => return Ok(false), + }; + + if !must_check_website { + return Ok(true); + } let bucket = self .garage @@ -119,20 +161,12 @@ impl AdminApiServer { let bucket_website_config = bucket_state.website_config.get(); match bucket_website_config { - Some(_v) => { - Ok(Response::builder() - .status(StatusCode::OK) - .body(Body::from(format!( - "Bucket '{domain}' is authorized for website hosting" - )))?) - } - None => Err(Error::bad_request(format!( - "Bucket '{domain}' is not authorized for website hosting" - ))), + Some(_v) => Ok(true), + None => Ok(false), } } - fn handle_health(&self) -> Result, Error> { + fn handle_health(&self) -> Result, Error> { let health = self.garage.system.health(); let (status, status_str) = match health.status { @@ -147,17 +181,17 @@ impl AdminApiServer { ), }; let status_str = format!( - "{}\nConsult the full health check API endpoint at /v0/health for more details\n", + "{}\nConsult the full health check API endpoint at /v1/health for more details\n", status_str ); Ok(Response::builder() .status(status) .header(http::header::CONTENT_TYPE, "text/plain") - .body(Body::from(status_str))?) + .body(string_body(status_str))?) } - fn handle_metrics(&self) -> Result, Error> { + fn handle_metrics(&self) -> Result, Error> { #[cfg(feature = "metrics")] { use opentelemetry::trace::Tracer; @@ -177,7 +211,7 @@ impl AdminApiServer { Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, encoder.format_type()) - .body(Body::from(buffer))?) + .body(bytes_body(buffer.into()))?) } #[cfg(not(feature = "metrics"))] Err(Error::bad_request( @@ -186,7 +220,6 @@ impl AdminApiServer { } } -#[async_trait] impl ApiHandler for AdminApiServer { const API_NAME: &'static str = "admin"; const API_NAME_DISPLAY: &'static str = "Admin"; @@ -194,20 +227,25 @@ impl ApiHandler for AdminApiServer { type Endpoint = Endpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { - Endpoint::from_request(req) + fn parse_endpoint(&self, req: &Request) -> Result { + if req.uri().path().starts_with("/v0/") { + let endpoint_v0 = router_v0::Endpoint::from_request(req)?; + Endpoint::from_v0(endpoint_v0) + } else { + Endpoint::from_request(req) + } } async fn handle( &self, - req: Request, + req: Request, endpoint: Endpoint, - ) -> Result, Error> { - let expected_auth_header = + ) -> Result, Error> { + let required_auth_hash = match endpoint.authorization_type() { Authorization::None => None, - Authorization::MetricsToken => self.metrics_token.as_ref(), - Authorization::AdminToken => match &self.admin_token { + Authorization::MetricsToken => self.metrics_token.as_deref(), + Authorization::AdminToken => match self.admin_token.as_deref() { None => return Err(Error::forbidden( "Admin token isn't configured, admin API access is disabled for security.", )), @@ -215,21 +253,18 @@ impl ApiHandler for AdminApiServer { }, }; - if let Some(h) = expected_auth_header { + if let Some(password_hash) = required_auth_hash { match req.headers().get("Authorization") { None => return Err(Error::forbidden("Authorization token must be provided")), - Some(v) => { - let authorized = v.to_str().map(|hv| hv.trim() == h).unwrap_or(false); - if !authorized { - return Err(Error::forbidden("Invalid authorization token provided")); - } + Some(authorization) => { + verify_bearer_token(&authorization, password_hash)?; } } } match endpoint { Endpoint::Options => self.handle_options(&req), - Endpoint::CheckWebsiteEnabled => self.handle_check_website_enabled(req).await, + Endpoint::CheckDomain => self.handle_check_domain(req).await, Endpoint::Health => self.handle_health(), Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, @@ -239,11 +274,16 @@ impl ApiHandler for AdminApiServer { Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await, - Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await, + Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage).await, // Keys Endpoint::ListKeys => handle_list_keys(&self.garage).await, - Endpoint::GetKeyInfo { id, search } => { - handle_get_key_info(&self.garage, id, search).await + Endpoint::GetKeyInfo { + id, + search, + show_secret_key, + } => { + let show_secret_key = show_secret_key.map(|x| x == "true").unwrap_or(false); + handle_get_key_info(&self.garage, id, search, show_secret_key).await } Endpoint::CreateKey => handle_create_key(&self.garage, req).await, Endpoint::ImportKey => handle_import_key(&self.garage, req).await, @@ -292,3 +332,35 @@ impl ApiEndpoint for Endpoint { fn add_span_attributes(&self, _span: SpanRef<'_>) {} } + +fn hash_bearer_token(token: &str) -> String { + use argon2::{ + password_hash::{rand_core::OsRng, PasswordHasher, SaltString}, + Argon2, + }; + + let salt = SaltString::generate(&mut OsRng); + let argon2 = Argon2::default(); + argon2 + .hash_password(token.trim().as_bytes(), &salt) + .expect("could not hash API token") + .to_string() +} + +fn verify_bearer_token(token: &hyper::http::HeaderValue, password_hash: &str) -> Result<(), Error> { + use argon2::{password_hash::PasswordVerifier, Argon2}; + + let parsed_hash = PasswordHash::new(&password_hash).unwrap(); + + token + .to_str()? + .strip_prefix("Bearer ") + .and_then(|token| { + Argon2::default() + .verify_password(token.trim().as_bytes(), &parsed_hash) + .ok() + }) + .ok_or_else(|| Error::forbidden("Invalid authorization token"))?; + + Ok(()) +} diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index e60f07ca..207693b6 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use garage_util::crdt::*; @@ -14,14 +14,17 @@ use garage_model::bucket_alias_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_model::permission::*; +use garage_model::s3::mpu_table; use garage_model::s3::object_table::*; -use crate::admin::error::*; -use crate::admin::key::ApiBucketKeyPerm; -use crate::common_error::CommonError; -use crate::helpers::{json_ok_response, parse_json_body}; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; -pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { +use crate::api_server::ResBody; +use crate::error::*; +use crate::key::ApiBucketKeyPerm; + +pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage .bucket_table .get_range( @@ -89,7 +92,7 @@ pub async fn handle_get_bucket_info( garage: &Arc, id: Option, global_alias: Option, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = match (id, global_alias) { (Some(id), None) => parse_bucket_id(&id)?, (None, Some(ga)) => garage @@ -110,7 +113,7 @@ pub async fn handle_get_bucket_info( async fn bucket_info_results( garage: &Arc, bucket_id: Uuid, -) -> Result, Error> { +) -> Result, Error> { let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) @@ -121,7 +124,15 @@ async fn bucket_info_results( .table .get(&bucket_id, &EmptyKey) .await? - .map(|x| x.filtered_values(&garage.system.ring.borrow())) + .map(|x| x.filtered_values(&garage.system.cluster_layout())) + .unwrap_or_default(); + + let mpu_counters = garage + .mpu_counter_table + .table + .get(&bucket_id, &EmptyKey) + .await? + .map(|x| x.filtered_values(&garage.system.cluster_layout())) .unwrap_or_default(); let mut relevant_keys = HashMap::new(); @@ -183,8 +194,8 @@ async fn bucket_info_results( } }), keys: relevant_keys - .into_iter() - .map(|(_, key)| { + .into_values() + .map(|key| { let p = key.state.as_option().unwrap(); GetBucketInfoKey { access_key_id: key.key_id, @@ -208,12 +219,12 @@ async fn bucket_info_results( } }) .collect::>(), - objects: counters.get(OBJECTS).cloned().unwrap_or_default(), - bytes: counters.get(BYTES).cloned().unwrap_or_default(), - unfinished_uploads: counters - .get(UNFINISHED_UPLOADS) - .cloned() - .unwrap_or_default(), + objects: *counters.get(OBJECTS).unwrap_or(&0), + bytes: *counters.get(BYTES).unwrap_or(&0), + unfinished_uploads: *counters.get(UNFINISHED_UPLOADS).unwrap_or(&0), + unfinished_multipart_uploads: *mpu_counters.get(mpu_table::UPLOADS).unwrap_or(&0), + unfinished_multipart_upload_parts: *mpu_counters.get(mpu_table::PARTS).unwrap_or(&0), + unfinished_multipart_upload_bytes: *mpu_counters.get(mpu_table::BYTES).unwrap_or(&0), quotas: ApiBucketQuotas { max_size: quotas.max_size, max_objects: quotas.max_objects, @@ -235,6 +246,9 @@ struct GetBucketInfoResult { objects: i64, bytes: i64, unfinished_uploads: i64, + unfinished_multipart_uploads: i64, + unfinished_multipart_upload_parts: i64, + unfinished_multipart_upload_bytes: i64, quotas: ApiBucketQuotas, } @@ -256,12 +270,14 @@ struct GetBucketInfoKey { pub async fn handle_create_bucket( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let helper = garage.locked_helper().await; if let Some(ga) = &req.global_alias { - if !is_valid_bucket_name(ga) { + if !is_valid_bucket_name(ga, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", ga, INVALID_BUCKET_NAME_MESSAGE @@ -276,17 +292,14 @@ pub async fn handle_create_bucket( } if let Some(la) = &req.local_alias { - if !is_valid_bucket_name(&la.alias) { + if !is_valid_bucket_name(&la.alias, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", la.alias, INVALID_BUCKET_NAME_MESSAGE ))); } - let key = garage - .key_helper() - .get_existing_key(&la.access_key_id) - .await?; + let key = helper.key().get_existing_key(&la.access_key_id).await?; let state = key.state.as_option().unwrap(); if matches!(state.local_aliases.get(&la.alias), Some(_)) { return Err(Error::bad_request("Local alias already exists")); @@ -297,21 +310,16 @@ pub async fn handle_create_bucket( garage.bucket_table.insert(&bucket).await?; if let Some(ga) = &req.global_alias { - garage - .bucket_helper() - .set_global_bucket_alias(bucket.id, ga) - .await?; + helper.set_global_bucket_alias(bucket.id, ga).await?; } if let Some(la) = &req.local_alias { - garage - .bucket_helper() + helper .set_local_bucket_alias(bucket.id, &la.access_key_id, &la.alias) .await?; if la.allow.read || la.allow.write || la.allow.owner { - garage - .bucket_helper() + helper .set_bucket_key_permissions( bucket.id, &la.access_key_id, @@ -348,16 +356,16 @@ struct CreateBucketLocalAlias { pub async fn handle_delete_bucket( garage: &Arc, id: String, -) -> Result, Error> { - let helper = garage.bucket_helper(); +) -> Result, Error> { + let helper = garage.locked_helper().await; let bucket_id = parse_bucket_id(&id)?; - let mut bucket = helper.get_existing_bucket(bucket_id).await?; + let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?; let state = bucket.state.as_option().unwrap(); // Check bucket is empty - if !helper.is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } @@ -374,7 +382,7 @@ pub async fn handle_delete_bucket( for ((key_id, alias), _, active) in state.local_aliases.items().iter() { if *active { helper - .unset_local_bucket_alias(bucket.id, key_id, alias) + .purge_local_bucket_alias(bucket.id, key_id, alias) .await?; } } @@ -391,15 +399,15 @@ pub async fn handle_delete_bucket( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_update_bucket( garage: &Arc, id: String, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let bucket_id = parse_bucket_id(&id)?; let mut bucket = garage @@ -458,23 +466,19 @@ struct UpdateBucketWebsiteAccess { pub async fn handle_bucket_change_key_perm( garage: &Arc, - req: Request, + req: Request, new_perm_flag: bool, -) -> Result, Error> { - let req = parse_json_body::(req).await?; +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let helper = garage.locked_helper().await; let bucket_id = parse_bucket_id(&req.bucket_id)?; - let bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; + let bucket = helper.bucket().get_existing_bucket(bucket_id).await?; let state = bucket.state.as_option().unwrap(); - let key = garage - .key_helper() - .get_existing_key(&req.access_key_id) - .await?; + let key = helper.key().get_existing_key(&req.access_key_id).await?; let mut perm = state .authorized_keys @@ -492,8 +496,7 @@ pub async fn handle_bucket_change_key_perm( perm.allow_owner = new_perm_flag; } - garage - .bucket_helper() + helper .set_bucket_key_permissions(bucket.id, &key.key_id, perm) .await?; @@ -514,13 +517,12 @@ pub async fn handle_global_alias_bucket( garage: &Arc, bucket_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() - .set_global_bucket_alias(bucket_id, &alias) - .await?; + let helper = garage.locked_helper().await; + + helper.set_global_bucket_alias(bucket_id, &alias).await?; bucket_info_results(garage, bucket_id).await } @@ -529,13 +531,12 @@ pub async fn handle_global_unalias_bucket( garage: &Arc, bucket_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &alias) - .await?; + let helper = garage.locked_helper().await; + + helper.unset_global_bucket_alias(bucket_id, &alias).await?; bucket_info_results(garage, bucket_id).await } @@ -545,11 +546,12 @@ pub async fn handle_local_alias_bucket( bucket_id: String, access_key_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() + let helper = garage.locked_helper().await; + + helper .set_local_bucket_alias(bucket_id, &access_key_id, &alias) .await?; @@ -561,11 +563,12 @@ pub async fn handle_local_unalias_bucket( bucket_id: String, access_key_id: String, alias: String, -) -> Result, Error> { +) -> Result, Error> { let bucket_id = parse_bucket_id(&bucket_id)?; - garage - .bucket_helper() + let helper = garage.locked_helper().await; + + helper .unset_local_bucket_alias(bucket_id, &access_key_id, &alias) .await?; diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 98bf2b5a..ffa0fa71 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -2,58 +2,145 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response}; use serde::{Deserialize, Serialize}; use garage_util::crdt::*; use garage_util::data::*; -use garage_rpc::layout::*; +use garage_rpc::layout; use garage_model::garage::Garage; -use crate::admin::error::*; -use crate::helpers::{json_ok_response, parse_json_body}; +use garage_api_common::helpers::{json_ok_response, parse_json_body}; + +use crate::api_server::ResBody; +use crate::error::*; + +pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { + let layout = garage.system.cluster_layout(); + let mut nodes = garage + .system + .get_known_nodes() + .into_iter() + .map(|i| { + ( + i.id, + NodeResp { + id: hex::encode(i.id), + addr: i.addr, + hostname: i.status.hostname, + is_up: i.is_up, + last_seen_secs_ago: i.last_seen_secs_ago, + data_partition: i + .status + .data_disk_avail + .map(|(avail, total)| FreeSpaceResp { + available: avail, + total, + }), + metadata_partition: i.status.meta_disk_avail.map(|(avail, total)| { + FreeSpaceResp { + available: avail, + total, + } + }), + ..Default::default() + }, + ) + }) + .collect::>(); + + for (id, _, role) in layout.current().roles.items().iter() { + if let layout::NodeRoleV(Some(r)) = role { + let role = NodeRoleResp { + id: hex::encode(id), + zone: r.zone.to_string(), + capacity: r.capacity, + tags: r.tags.clone(), + }; + match nodes.get_mut(id) { + None => { + nodes.insert( + *id, + NodeResp { + id: hex::encode(id), + role: Some(role), + ..Default::default() + }, + ); + } + Some(n) => { + n.role = Some(role); + } + } + } + } + + for ver in layout.versions().iter().rev().skip(1) { + for (id, _, role) in ver.roles.items().iter() { + if let layout::NodeRoleV(Some(r)) = role { + if r.capacity.is_some() { + if let Some(n) = nodes.get_mut(id) { + if n.role.is_none() { + n.draining = true; + } + } else { + nodes.insert( + *id, + NodeResp { + id: hex::encode(id), + draining: true, + ..Default::default() + }, + ); + } + } + } + } + } + + let mut nodes = nodes.into_values().collect::>(); + nodes.sort_by(|x, y| x.id.cmp(&y.id)); -pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { node: hex::encode(garage.system.id), garage_version: garage_util::version::garage_version(), garage_features: garage_util::version::garage_features(), rust_version: garage_util::version::rust_version(), db_engine: garage.db.engine(), - known_nodes: garage - .system - .get_known_nodes() - .into_iter() - .map(|i| { - ( - hex::encode(i.id), - KnownNodeResp { - addr: i.addr, - is_up: i.is_up, - last_seen_secs_ago: i.last_seen_secs_ago, - hostname: i.status.hostname, - }, - ) - }) - .collect(), - layout: get_cluster_layout(garage), + layout_version: layout.current().version, + nodes, }; Ok(json_ok_response(&res)?) } -pub async fn handle_get_cluster_health(garage: &Arc) -> Result, Error> { +pub async fn handle_get_cluster_health(garage: &Arc) -> Result, Error> { + use garage_rpc::system::ClusterHealthStatus; let health = garage.system.health(); + let health = ClusterHealth { + status: match health.status { + ClusterHealthStatus::Healthy => "healthy", + ClusterHealthStatus::Degraded => "degraded", + ClusterHealthStatus::Unavailable => "unavailable", + }, + known_nodes: health.known_nodes, + connected_nodes: health.connected_nodes, + storage_nodes: health.storage_nodes, + storage_nodes_ok: health.storage_nodes_ok, + partitions: health.partitions, + partitions_quorum: health.partitions_quorum, + partitions_all_ok: health.partitions_all_ok, + }; Ok(json_ok_response(&health)?) } pub async fn handle_connect_cluster_nodes( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::>(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::, _, Error>(req).await?; let res = futures::future::join_all(req.iter().map(|node| garage.system.connect(node))) .await @@ -73,34 +160,72 @@ pub async fn handle_connect_cluster_nodes( Ok(json_ok_response(&res)?) } -pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { - let res = get_cluster_layout(garage); +pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { + let res = format_cluster_layout(garage.system.cluster_layout().inner()); Ok(json_ok_response(&res)?) } -fn get_cluster_layout(garage: &Arc) -> GetClusterLayoutResponse { - let layout = garage.system.get_cluster_layout(); +fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResponse { + let roles = layout + .current() + .roles + .items() + .iter() + .filter_map(|(k, _, v)| v.0.clone().map(|x| (k, x))) + .map(|(k, v)| NodeRoleResp { + id: hex::encode(k), + zone: v.zone.clone(), + capacity: v.capacity, + tags: v.tags.clone(), + }) + .collect::>(); + + let staged_role_changes = layout + .staging + .get() + .roles + .items() + .iter() + .filter(|(k, _, v)| layout.current().roles.get(k) != Some(v)) + .map(|(k, _, v)| match &v.0 { + None => NodeRoleChange { + id: hex::encode(k), + action: NodeRoleChangeEnum::Remove { remove: true }, + }, + Some(r) => NodeRoleChange { + id: hex::encode(k), + action: NodeRoleChangeEnum::Update { + zone: r.zone.clone(), + capacity: r.capacity, + tags: r.tags.clone(), + }, + }, + }) + .collect::>(); GetClusterLayoutResponse { - version: layout.version, - roles: layout - .roles - .items() - .iter() - .filter(|(_, _, v)| v.0.is_some()) - .map(|(k, _, v)| (hex::encode(k), v.0.clone())) - .collect(), - staged_role_changes: layout - .staging - .items() - .iter() - .filter(|(k, _, v)| layout.roles.get(k) != Some(v)) - .map(|(k, _, v)| (hex::encode(k), v.0.clone())) - .collect(), + version: layout.current().version, + roles, + staged_role_changes, } } +// ---- + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClusterHealth { + status: &'static str, + known_nodes: usize, + connected_nodes: usize, + storage_nodes: usize, + storage_nodes_ok: usize, + partitions: usize, + partitions_quorum: usize, + partitions_all_ok: usize, +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct GetClusterStatusResponse { @@ -109,11 +234,19 @@ struct GetClusterStatusResponse { garage_features: Option<&'static [&'static str]>, rust_version: &'static str, db_engine: String, - known_nodes: HashMap, + layout_version: u64, + nodes: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ApplyClusterLayoutResponse { + message: Vec, layout: GetClusterLayoutResponse, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct ConnectClusterNodesResponse { success: bool, error: Option, @@ -123,78 +256,156 @@ struct ConnectClusterNodesResponse { #[serde(rename_all = "camelCase")] struct GetClusterLayoutResponse { version: u64, - roles: HashMap>, - staged_role_changes: HashMap>, + roles: Vec, + staged_role_changes: Vec, } #[derive(Serialize)] -struct KnownNodeResp { - addr: SocketAddr, +#[serde(rename_all = "camelCase")] +struct NodeRoleResp { + id: String, + zone: String, + capacity: Option, + tags: Vec, +} + +#[derive(Serialize, Default)] +#[serde(rename_all = "camelCase")] +struct FreeSpaceResp { + available: u64, + total: u64, +} + +#[derive(Serialize, Default)] +#[serde(rename_all = "camelCase")] +struct NodeResp { + id: String, + role: Option, + addr: Option, + hostname: Option, is_up: bool, last_seen_secs_ago: Option, - hostname: String, + draining: bool, + #[serde(skip_serializing_if = "Option::is_none")] + data_partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + metadata_partition: Option, } +// ---- update functions ---- + pub async fn handle_update_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let updates = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let updates = parse_json_body::(req).await?; - let mut layout = garage.system.get_cluster_layout(); + let mut layout = garage.system.cluster_layout().inner().clone(); - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); - for (node, role) in updates { - let node = hex::decode(node).ok_or_bad_request("Invalid node identifier")?; + for change in updates { + let node = hex::decode(&change.id).ok_or_bad_request("Invalid node identifier")?; let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?; + let new_role = match change.action { + NodeRoleChangeEnum::Remove { remove: true } => None, + NodeRoleChangeEnum::Update { + zone, + capacity, + tags, + } => Some(layout::NodeRole { + zone, + capacity, + tags, + }), + _ => return Err(Error::bad_request("Invalid layout change")), + }; + layout .staging - .merge(&roles.update_mutator(node, NodeRoleV(role))); + .get_mut() + .roles + .merge(&roles.update_mutator(node, layout::NodeRoleV(new_role))); } - garage.system.update_cluster_layout(&layout).await?; + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; - Ok(Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + let res = format_cluster_layout(&layout); + Ok(json_ok_response(&res)?) } pub async fn handle_apply_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let param = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let param = parse_json_body::(req).await?; - let layout = garage.system.get_cluster_layout(); - let layout = layout.apply_staged_changes(Some(param.version))?; - garage.system.update_cluster_layout(&layout).await?; + let layout = garage.system.cluster_layout().inner().clone(); + let (layout, msg) = layout.apply_staged_changes(Some(param.version))?; - Ok(Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; + + let res = ApplyClusterLayoutResponse { + message: msg, + layout: format_cluster_layout(&layout), + }; + Ok(json_ok_response(&res)?) } pub async fn handle_revert_cluster_layout( garage: &Arc, - req: Request, -) -> Result, Error> { - let param = parse_json_body::(req).await?; +) -> Result, Error> { + let layout = garage.system.cluster_layout().inner().clone(); + let layout = layout.revert_staged_changes()?; + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; - let layout = garage.system.get_cluster_layout(); - let layout = layout.revert_staged_changes(Some(param.version))?; - garage.system.update_cluster_layout(&layout).await?; - - Ok(Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + let res = format_cluster_layout(&layout); + Ok(json_ok_response(&res)?) } -type UpdateClusterLayoutRequest = HashMap>; +// ---- + +type UpdateClusterLayoutRequest = Vec; #[derive(Deserialize)] -struct ApplyRevertLayoutRequest { +#[serde(rename_all = "camelCase")] +struct ApplyLayoutRequest { version: u64, } + +// ---- + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct NodeRoleChange { + id: String, + #[serde(flatten)] + action: NodeRoleChangeEnum, +} + +#[derive(Serialize, Deserialize)] +#[serde(untagged)] +enum NodeRoleChangeEnum { + #[serde(rename_all = "camelCase")] + Remove { remove: bool }, + #[serde(rename_all = "camelCase")] + Update { + zone: String, + capacity: Option, + tags: Vec, + }, +} diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index ed1a07bd..17d4c200 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -1,53 +1,46 @@ -use err_derive::Error; +use std::convert::TryFrom; + use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; pub use garage_model::helper::error::Error as HelperError; -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::CustomApiErrorBody; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// The API access key does not exist - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), /// In Import key, the key already exists - #[error( - display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", - _0 - )] + #[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")] KeyAlreadyExists(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} +commonErrorDerivative!(Error); +/// FIXME: helper errors are transformed into their corresponding variants +/// in the Error struct, but in many case a helper error should be considered +/// an internal error. impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n), + fn from(err: HelperError) -> Error { + match CommonError::try_from(err) { + Ok(ce) => Self::Common(ce), + Err(HelperError::NoSuchAccessKey(k)) => Self::NoSuchAccessKey(k), + Err(_) => unreachable!(), } } } @@ -77,14 +70,14 @@ impl ApiError for Error { header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap()); } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = CustomApiErrorBody { code: self.code().to_string(), message: format!("{}", self), path: path.to_string(), region: garage_region.to_string(), }; - Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + let error_str = serde_json::to_string_pretty(&error).unwrap_or_else(|_| { r#" { "code": "InternalError", @@ -92,6 +85,7 @@ impl ApiError for Error { } "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 2bbabb7b..bebf3063 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use garage_table::*; @@ -9,10 +9,12 @@ use garage_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; -use crate::admin::error::*; -use crate::helpers::{json_ok_response, parse_json_body}; +use garage_api_common::helpers::*; -pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { +use crate::api_server::ResBody; +use crate::error::*; + +pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage .key_table .get_range( @@ -34,6 +36,7 @@ pub async fn handle_list_keys(garage: &Arc) -> Result, Er } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct ListKeyResultItem { id: String, name: String, @@ -43,7 +46,8 @@ pub async fn handle_get_key_info( garage: &Arc, id: Option, search: Option, -) -> Result, Error> { + show_secret_key: bool, +) -> Result, Error> { let key = if let Some(id) = id { garage.key_helper().get_existing_key(&id).await? } else if let Some(search) = search { @@ -55,41 +59,47 @@ pub async fn handle_get_key_info( unreachable!(); }; - key_info_results(garage, key).await + key_info_results(garage, key, show_secret_key).await } pub async fn handle_create_key( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; - let key = Key::new(&req.name); + let key = Key::new(req.name.as_deref().unwrap_or("Unnamed key")); garage.key_table.insert(&key).await?; - key_info_results(garage, key).await + key_info_results(garage, key, true).await } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct CreateKeyRequest { - name: String, + name: Option, } pub async fn handle_import_key( garage: &Arc, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let prev_key = garage.key_table.get(&EmptyKey, &req.access_key_id).await?; if prev_key.is_some() { return Err(Error::KeyAlreadyExists(req.access_key_id.to_string())); } - let imported_key = Key::import(&req.access_key_id, &req.secret_access_key, &req.name); + let imported_key = Key::import( + &req.access_key_id, + &req.secret_access_key, + req.name.as_deref().unwrap_or("Imported key"), + ) + .ok_or_bad_request("Invalid key format")?; garage.key_table.insert(&imported_key).await?; - key_info_results(garage, imported_key).await + key_info_results(garage, imported_key, false).await } #[derive(Deserialize)] @@ -97,15 +107,15 @@ pub async fn handle_import_key( struct ImportKeyRequest { access_key_id: String, secret_access_key: String, - name: String, + name: Option, } pub async fn handle_update_key( garage: &Arc, id: String, - req: Request, -) -> Result, Error> { - let req = parse_json_body::(req).await?; + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; let mut key = garage.key_helper().get_existing_key(&id).await?; @@ -127,29 +137,37 @@ pub async fn handle_update_key( garage.key_table.insert(&key).await?; - key_info_results(garage, key).await + key_info_results(garage, key, false).await } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct UpdateKeyRequest { name: Option, allow: Option, deny: Option, } -pub async fn handle_delete_key(garage: &Arc, id: String) -> Result, Error> { - let mut key = garage.key_helper().get_existing_key(&id).await?; +pub async fn handle_delete_key( + garage: &Arc, + id: String, +) -> Result, Error> { + let helper = garage.locked_helper().await; - key.state.as_option().unwrap(); + let mut key = helper.key().get_existing_key(&id).await?; - garage.key_helper().delete_key(&mut key).await?; + helper.delete_key(&mut key).await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } -async fn key_info_results(garage: &Arc, key: Key) -> Result, Error> { +async fn key_info_results( + garage: &Arc, + key: Key, + show_secret: bool, +) -> Result, Error> { let mut relevant_buckets = HashMap::new(); let key_state = key.state.as_option().unwrap(); @@ -178,13 +196,17 @@ async fn key_info_results(garage: &Arc, key: Key) -> Result, key: Key) -> Result, permissions: KeyPerm, buckets: Vec, } @@ -246,6 +269,7 @@ struct KeyInfoBucketResult { } #[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] pub(crate) struct ApiBucketKeyPerm { #[serde(default)] pub(crate) read: bool, diff --git a/src/api/admin/mod.rs b/src/api/admin/lib.rs similarity index 50% rename from src/api/admin/mod.rs rename to src/api/admin/lib.rs index c4857c10..599e9b44 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/lib.rs @@ -1,6 +1,10 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; mod error; -mod router; +mod router_v0; +mod router_v1; mod bucket; mod cluster; diff --git a/src/api/admin/router.rs b/src/api/admin/router_v0.rs similarity index 87% rename from src/api/admin/router.rs rename to src/api/admin/router_v0.rs index 0dcb1546..9dd742ba 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router_v0.rs @@ -2,14 +2,9 @@ use std::borrow::Cow; use hyper::{Method, Request}; -use crate::admin::error::*; -use crate::router_macros::*; +use garage_api_common::router_macros::*; -pub enum Authorization { - None, - MetricsToken, - AdminToken, -} +use crate::error::*; router_match! {@func @@ -17,7 +12,7 @@ router_match! {@func #[derive(Debug, Clone, PartialEq, Eq)] pub enum Endpoint { Options, - CheckWebsiteEnabled, + CheckDomain, Health, Metrics, GetClusterStatus, @@ -92,7 +87,7 @@ impl Endpoint { let res = router_match!(@gen_path_parser (req.method(), path, query) [ OPTIONS _ => Options, - GET "/check" => CheckWebsiteEnabled, + GET "/check" => CheckDomain, GET "/health" => Health, GET "/metrics" => Metrics, GET "/v0/status" => GetClusterStatus, @@ -134,15 +129,6 @@ impl Endpoint { Ok(res) } - /// Get the kind of authorization which is required to perform the operation. - pub fn authorization_type(&self) -> Authorization { - match self { - Self::Health => Authorization::None, - Self::CheckWebsiteEnabled => Authorization::None, - Self::Metrics => Authorization::MetricsToken, - _ => Authorization::AdminToken, - } - } } generateQueryParameters! { diff --git a/src/api/admin/router_v1.rs b/src/api/admin/router_v1.rs new file mode 100644 index 00000000..0b4901ea --- /dev/null +++ b/src/api/admin/router_v1.rs @@ -0,0 +1,236 @@ +use std::borrow::Cow; + +use hyper::{Method, Request}; + +use garage_api_common::router_macros::*; + +use crate::error::*; +use crate::router_v0; + +pub enum Authorization { + None, + MetricsToken, + AdminToken, +} + +router_match! {@func + +/// List of all Admin API endpoints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Endpoint { + Options, + CheckDomain, + Health, + Metrics, + GetClusterStatus, + GetClusterHealth, + ConnectClusterNodes, + // Layout + GetClusterLayout, + UpdateClusterLayout, + ApplyClusterLayout, + RevertClusterLayout, + // Keys + ListKeys, + CreateKey, + ImportKey, + GetKeyInfo { + id: Option, + search: Option, + show_secret_key: Option, + }, + DeleteKey { + id: String, + }, + UpdateKey { + id: String, + }, + // Buckets + ListBuckets, + CreateBucket, + GetBucketInfo { + id: Option, + global_alias: Option, + }, + DeleteBucket { + id: String, + }, + UpdateBucket { + id: String, + }, + // Bucket-Key Permissions + BucketAllowKey, + BucketDenyKey, + // Bucket aliases + GlobalAliasBucket { + id: String, + alias: String, + }, + GlobalUnaliasBucket { + id: String, + alias: String, + }, + LocalAliasBucket { + id: String, + access_key_id: String, + alias: String, + }, + LocalUnaliasBucket { + id: String, + access_key_id: String, + alias: String, + }, +}} + +impl Endpoint { + /// Determine which S3 endpoint a request is for using the request, and a bucket which was + /// possibly extracted from the Host header. + /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets + pub fn from_request(req: &Request) -> Result { + let uri = req.uri(); + let path = uri.path(); + let query = uri.query(); + + let mut query = QueryParameters::from_query(query.unwrap_or_default())?; + + let res = router_match!(@gen_path_parser (req.method(), path, query) [ + OPTIONS _ => Options, + GET "/check" => CheckDomain, + GET "/health" => Health, + GET "/metrics" => Metrics, + GET "/v1/status" => GetClusterStatus, + GET "/v1/health" => GetClusterHealth, + POST "/v1/connect" => ConnectClusterNodes, + // Layout endpoints + GET "/v1/layout" => GetClusterLayout, + POST "/v1/layout" => UpdateClusterLayout, + POST "/v1/layout/apply" => ApplyClusterLayout, + POST "/v1/layout/revert" => RevertClusterLayout, + // API key endpoints + GET "/v1/key" if id => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), + GET "/v1/key" if search => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key), + POST "/v1/key" if id => UpdateKey (query::id), + POST "/v1/key" => CreateKey, + POST "/v1/key/import" => ImportKey, + DELETE "/v1/key" if id => DeleteKey (query::id), + GET "/v1/key" => ListKeys, + // Bucket endpoints + GET "/v1/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v1/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v1/bucket" => ListBuckets, + POST "/v1/bucket" => CreateBucket, + DELETE "/v1/bucket" if id => DeleteBucket (query::id), + PUT "/v1/bucket" if id => UpdateBucket (query::id), + // Bucket-key permissions + POST "/v1/bucket/allow" => BucketAllowKey, + POST "/v1/bucket/deny" => BucketDenyKey, + // Bucket aliases + PUT "/v1/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), + DELETE "/v1/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), + PUT "/v1/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), + DELETE "/v1/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), + ]); + + if let Some(message) = query.nonempty_message() { + debug!("Unused query parameter: {}", message) + } + + Ok(res) + } + /// Some endpoints work exactly the same in their v1/ version as they did in their v0/ version. + /// For these endpoints, we can convert a v0/ call to its equivalent as if it was made using + /// its v1/ URL. + pub fn from_v0(v0_endpoint: router_v0::Endpoint) -> Result { + match v0_endpoint { + // Cluster endpoints + router_v0::Endpoint::ConnectClusterNodes => Ok(Self::ConnectClusterNodes), + // - GetClusterStatus: response format changed + // - GetClusterHealth: response format changed + + // Layout endpoints + router_v0::Endpoint::RevertClusterLayout => Ok(Self::RevertClusterLayout), + // - GetClusterLayout: response format changed + // - UpdateClusterLayout: query format changed + // - ApplyCusterLayout: response format changed + + // Key endpoints + router_v0::Endpoint::ListKeys => Ok(Self::ListKeys), + router_v0::Endpoint::CreateKey => Ok(Self::CreateKey), + router_v0::Endpoint::GetKeyInfo { id, search } => Ok(Self::GetKeyInfo { + id, + search, + show_secret_key: Some("true".into()), + }), + router_v0::Endpoint::DeleteKey { id } => Ok(Self::DeleteKey { id }), + // - UpdateKey: response format changed (secret key no longer returned) + + // Bucket endpoints + router_v0::Endpoint::GetBucketInfo { id, global_alias } => { + Ok(Self::GetBucketInfo { id, global_alias }) + } + router_v0::Endpoint::ListBuckets => Ok(Self::ListBuckets), + router_v0::Endpoint::CreateBucket => Ok(Self::CreateBucket), + router_v0::Endpoint::DeleteBucket { id } => Ok(Self::DeleteBucket { id }), + router_v0::Endpoint::UpdateBucket { id } => Ok(Self::UpdateBucket { id }), + + // Bucket-key permissions + router_v0::Endpoint::BucketAllowKey => Ok(Self::BucketAllowKey), + router_v0::Endpoint::BucketDenyKey => Ok(Self::BucketDenyKey), + + // Bucket alias endpoints + router_v0::Endpoint::GlobalAliasBucket { id, alias } => { + Ok(Self::GlobalAliasBucket { id, alias }) + } + router_v0::Endpoint::GlobalUnaliasBucket { id, alias } => { + Ok(Self::GlobalUnaliasBucket { id, alias }) + } + router_v0::Endpoint::LocalAliasBucket { + id, + access_key_id, + alias, + } => Ok(Self::LocalAliasBucket { + id, + access_key_id, + alias, + }), + router_v0::Endpoint::LocalUnaliasBucket { + id, + access_key_id, + alias, + } => Ok(Self::LocalUnaliasBucket { + id, + access_key_id, + alias, + }), + + // For endpoints that have different body content syntax, issue + // deprecation warning + _ => Err(Error::bad_request(format!( + "v0/ endpoint is no longer supported: {}", + v0_endpoint.name() + ))), + } + } + /// Get the kind of authorization which is required to perform the operation. + pub fn authorization_type(&self) -> Authorization { + match self { + Self::Health => Authorization::None, + Self::CheckDomain => Authorization::None, + Self::Metrics => Authorization::MetricsToken, + _ => Authorization::AdminToken, + } + } +} + +generateQueryParameters! { + keywords: [], + fields: [ + "format" => format, + "id" => id, + "search" => search, + "globalAlias" => global_alias, + "alias" => alias, + "accessKeyId" => access_key_id, + "showSecretKey" => show_secret_key + ] +} diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml new file mode 100644 index 00000000..df01d59a --- /dev/null +++ b/src/api/common/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "garage_api_common" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Common functions for the API server crates for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_util.workspace = true + +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +thiserror.workspace = true +hex.workspace = true +hmac.workspace = true +md-5.workspace = true +tracing.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/common_error.rs b/src/api/common/common_error.rs similarity index 58% rename from src/api/common_error.rs rename to src/api/common/common_error.rs index 20f9f266..e596a6e9 100644 --- a/src/api/common_error.rs +++ b/src/api/common/common_error.rs @@ -1,61 +1,96 @@ -use err_derive::Error; +use std::convert::TryFrom; + use hyper::StatusCode; +use thiserror::Error; use garage_util::error::Error as GarageError; +use garage_model::helper::error::Error as HelperError; + /// Errors of this crate #[derive(Debug, Error)] pub enum CommonError { // ---- INTERNAL ERRORS ---- /// Error related to deeper parts of Garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), + #[error("Internal error: {0}")] + InternalError(#[from] GarageError), /// Error related to Hyper - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Internal error (Hyper error): {0}")] + Hyper(#[from] hyper::Error), /// Error related to HTTP - #[error(display = "Internal error (HTTP error): {}", _0)] - Http(#[error(source)] http::Error), + #[error("Internal error (HTTP error): {0}")] + Http(#[from] http::Error), // ---- GENERIC CLIENT ERRORS ---- /// Proper authentication was not provided - #[error(display = "Forbidden: {}", _0)] + #[error("Forbidden: {0}")] Forbidden(String), /// Generic bad request response with custom message - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), + /// The client sent a header with invalid value + #[error("Invalid header value: {0}")] + InvalidHeader(#[from] hyper::header::ToStrError), + // ---- SPECIFIC ERROR CONDITIONS ---- // These have to be error codes referenced in the S3 spec here: // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList /// The bucket requested don't exists - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] + #[error("Bucket already exists")] BucketAlreadyExists, /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] + #[error("Tried to delete a non-empty bucket")] BucketNotEmpty, // Category: bad request /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), } +#[macro_export] +macro_rules! commonErrorDerivative { + ( $error_struct: ident ) => { + impl From for $error_struct { + fn from(err: garage_util::error::Error) -> Self { + Self::Common(CommonError::InternalError(err)) + } + } + impl From for $error_struct { + fn from(err: http::Error) -> Self { + Self::Common(CommonError::Http(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::Error) -> Self { + Self::Common(CommonError::Hyper(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::header::ToStrError) -> Self { + Self::Common(CommonError::InvalidHeader(err)) + } + } + impl CommonErrorDerivative for $error_struct {} + }; +} + +pub use commonErrorDerivative; + impl CommonError { pub fn http_status_code(&self) -> StatusCode { match self { CommonError::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), ) => StatusCode::SERVICE_UNAVAILABLE, CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { StatusCode::INTERNAL_SERVER_ERROR @@ -64,7 +99,9 @@ impl CommonError { CommonError::Forbidden(_) => StatusCode::FORBIDDEN, CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, - CommonError::InvalidBucketName(_) => StatusCode::BAD_REQUEST, + CommonError::InvalidBucketName(_) | CommonError::InvalidHeader(_) => { + StatusCode::BAD_REQUEST + } } } @@ -72,9 +109,7 @@ impl CommonError { match self { CommonError::Forbidden(_) => "AccessDenied", CommonError::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), ) => "ServiceUnavailable", CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { "InternalError" @@ -84,6 +119,7 @@ impl CommonError { CommonError::BucketAlreadyExists => "BucketAlreadyExists", CommonError::BucketNotEmpty => "BucketNotEmpty", CommonError::InvalidBucketName(_) => "InvalidBucketName", + CommonError::InvalidHeader(_) => "InvalidHeaderValue", } } @@ -92,6 +128,39 @@ impl CommonError { } } +impl TryFrom for CommonError { + type Error = HelperError; + + fn try_from(err: HelperError) -> Result { + match err { + HelperError::Internal(i) => Ok(Self::InternalError(i)), + HelperError::BadRequest(b) => Ok(Self::BadRequest(b)), + HelperError::InvalidBucketName(n) => Ok(Self::InvalidBucketName(n)), + HelperError::NoSuchBucket(n) => Ok(Self::NoSuchBucket(n)), + e => Err(e), + } + } +} + +/// This function converts HelperErrors into CommonErrors, +/// for variants that exist in CommonError. +/// This is used for helper functions that might return InvalidBucketName +/// or NoSuchBucket for instance, and we want to pass that error +/// up to our caller. +pub fn pass_helper_error(err: HelperError) -> CommonError { + match CommonError::try_from(err) { + Ok(e) => e, + Err(e) => panic!("Helper error `{}` should hot have happenned here", e), + } +} + +pub fn helper_error_as_internal(err: HelperError) -> CommonError { + match err { + HelperError::Internal(e) => CommonError::InternalError(e), + e => CommonError::InternalError(GarageError::Message(e.to_string())), + } +} + pub trait CommonErrorDerivative: From { fn internal_error(msg: M) -> Self { Self::from(CommonError::InternalError(GarageError::Message( diff --git a/src/api/common/cors.rs b/src/api/common/cors.rs new file mode 100644 index 00000000..09b55c13 --- /dev/null +++ b/src/api/common/cors.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use http::header::{ + ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, + ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, +}; +use hyper::{body::Body, body::Incoming as IncomingBody, Request, Response, StatusCode}; + +use garage_model::bucket_table::{BucketParams, CorsRule as GarageCorsRule}; +use garage_model::garage::Garage; + +use crate::common_error::{ + helper_error_as_internal, CommonError, OkOrBadRequest, OkOrInternalError, +}; +use crate::helpers::*; + +pub fn find_matching_cors_rule<'a, B>( + bucket_params: &'a BucketParams, + req: &Request, +) -> Result, CommonError> { + if let Some(cors_config) = bucket_params.cors_config.get() { + if let Some(origin) = req.headers().get("Origin") { + let origin = origin.to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + return Ok(cors_config.iter().find(|rule| { + cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) + })); + } + } + Ok(None) +} + +pub fn cors_rule_matches<'a, HI, S>( + rule: &GarageCorsRule, + origin: &'a str, + method: &'a str, + mut request_headers: HI, +) -> bool +where + HI: Iterator, + S: AsRef, +{ + rule.allow_origins.iter().any(|x| x == "*" || x == origin) + && rule.allow_methods.iter().any(|x| x == "*" || x == method) + && request_headers.all(|h| { + rule.allow_headers + .iter() + .any(|x| x == "*" || x == h.as_ref()) + }) +} + +pub fn add_cors_headers( + resp: &mut Response, + rule: &GarageCorsRule, +) -> Result<(), http::header::InvalidHeaderValue> { + let h = resp.headers_mut(); + h.insert( + ACCESS_CONTROL_ALLOW_ORIGIN, + rule.allow_origins.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_METHODS, + rule.allow_methods.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_HEADERS, + rule.allow_headers.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_EXPOSE_HEADERS, + rule.expose_headers.join(", ").parse()?, + ); + Ok(()) +} + +pub async fn handle_options_api( + garage: Arc, + req: &Request, + bucket_name: Option, +) -> Result, CommonError> { + // FIXME: CORS rules of buckets with local aliases are + // not taken into account. + + // If the bucket name is a global bucket name, + // we try to apply the CORS rules of that bucket. + // If a user has a local bucket name that has + // the same name, its CORS rules won't be applied + // and will be shadowed by the rules of the globally + // existing bucket (but this is inevitable because + // OPTIONS calls are not auhtenticated). + if let Some(bn) = bucket_name { + let helper = garage.bucket_helper(); + let bucket_id = helper + .resolve_global_bucket_name(&bn) + .await + .map_err(helper_error_as_internal)?; + if let Some(id) = bucket_id { + let bucket = garage + .bucket_helper() + .get_existing_bucket(id) + .await + .map_err(helper_error_as_internal)?; + let bucket_params = bucket.state.into_option().unwrap(); + handle_options_for_bucket(req, &bucket_params) + } else { + // If there is a bucket name in the request, but that name + // does not correspond to a global alias for a bucket, + // then it's either a non-existing bucket or a local bucket. + // We have no way of knowing, because the request is not + // authenticated and thus we can't resolve local aliases. + // We take the permissive approach of allowing everything, + // because we don't want to prevent web apps that use + // local bucket names from making API calls. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "*") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } + } else { + // If there is no bucket name in the request, + // we are doing a ListBuckets call, which we want to allow + // for all origins. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } +} + +pub fn handle_options_for_bucket( + req: &Request, + bucket_params: &BucketParams, +) -> Result, CommonError> { + let origin = req + .headers() + .get("Origin") + .ok_or_bad_request("Missing Origin header")? + .to_str()?; + let request_method = req + .headers() + .get(ACCESS_CONTROL_REQUEST_METHOD) + .ok_or_bad_request("Missing Access-Control-Request-Method header")? + .to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + + if let Some(cors_config) = bucket_params.cors_config.get() { + let matching_rule = cors_config + .iter() + .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); + if let Some(rule) = matching_rule { + let mut resp = Response::builder() + .status(StatusCode::OK) + .body(EmptyBody::new())?; + add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; + return Ok(resp); + } + } + + Err(CommonError::Forbidden( + "This CORS request is not allowed.".into(), + )) +} diff --git a/src/api/encoding.rs b/src/api/common/encoding.rs similarity index 100% rename from src/api/encoding.rs rename to src/api/common/encoding.rs diff --git a/src/api/common/generic_server.rs b/src/api/common/generic_server.rs new file mode 100644 index 00000000..3f14c07d --- /dev/null +++ b/src/api/common/generic_server.rs @@ -0,0 +1,383 @@ +use std::convert::Infallible; +use std::fs::{self, Permissions}; +use std::os::unix::fs::PermissionsExt; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::Future; +use futures::stream::{futures_unordered::FuturesUnordered, StreamExt}; + +use http_body_util::BodyExt; +use hyper::header::HeaderValue; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{body::Incoming as IncomingBody, Request, Response}; +use hyper::{HeaderMap, StatusCode}; +use hyper_util::rt::TokioIo; + +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream}; +use tokio::sync::watch; +use tokio::time::{sleep_until, Instant}; + +use opentelemetry::{ + global, + metrics::{Counter, ValueRecorder}, + trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, + Context, KeyValue, +}; + +use garage_util::error::Error as GarageError; +use garage_util::forwarded_headers; +use garage_util::metrics::{gen_trace_id, RecordDuration}; +use garage_util::socket_address::UnixOrTCPSocketAddress; + +use crate::helpers::{BoxBody, ErrorBody}; + +pub trait ApiEndpoint: Send + Sync + 'static { + fn name(&self) -> &'static str; + fn add_span_attributes(&self, span: SpanRef<'_>); +} + +pub trait ApiError: std::error::Error + Send + Sync + 'static { + fn http_status_code(&self) -> StatusCode; + fn add_http_headers(&self, header_map: &mut HeaderMap); + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody; +} + +pub trait ApiHandler: Send + Sync + 'static { + const API_NAME: &'static str; + const API_NAME_DISPLAY: &'static str; + + type Endpoint: ApiEndpoint; + type Error: ApiError; + + fn parse_endpoint(&self, r: &Request) -> Result; + fn handle( + &self, + req: Request, + endpoint: Self::Endpoint, + ) -> impl Future>, Self::Error>> + Send; + + /// Returns the key id used to authenticate this request. The ID returned must be safe to + /// log. + fn key_id_from_request(&self, _req: &Request) -> Option { + None + } +} + +pub struct ApiServer { + region: String, + api_handler: A, + + // Metrics + request_counter: Counter, + error_counter: Counter, + request_duration: ValueRecorder, +} + +impl ApiServer { + pub fn new(region: String, api_handler: A) -> Arc { + let meter = global::meter("garage/api"); + Arc::new(Self { + region, + api_handler, + request_counter: meter + .u64_counter(format!("api.{}.request_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + error_counter: meter + .u64_counter(format!("api.{}.error_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints that resulted in errors", + A::API_NAME_DISPLAY + )) + .init(), + request_duration: meter + .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) + .with_description(format!( + "Duration of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + }) + } + + pub async fn run_server( + self: Arc, + bind_addr: UnixOrTCPSocketAddress, + unix_bind_addr_mode: Option, + must_exit: watch::Receiver, + ) -> Result<(), GarageError> { + let server_name = format!("{} API", A::API_NAME_DISPLAY); + info!("{} server listening on {}", server_name, bind_addr); + + match bind_addr { + UnixOrTCPSocketAddress::TCPSocket(addr) => { + let listener = TcpListener::bind(addr).await?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + UnixOrTCPSocketAddress::UnixSocket(ref path) => { + if path.exists() { + fs::remove_file(path)? + } + + let listener = UnixListener::bind(path)?; + let listener = UnixListenerOn(listener, path.display().to_string()); + + fs::set_permissions( + path, + Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)), + )?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + } + } + + async fn handler( + self: Arc, + req: Request, + addr: String, + ) -> Result>, http::Error> { + let uri = req.uri().clone(); + + let source = if let Ok(forwarded_for_ip_addr) = + forwarded_headers::handle_forwarded_for_headers(req.headers()) + { + format!("{forwarded_for_ip_addr} (via {addr})") + } else { + format!("{addr}") + }; + // we only do this to log the access key, so we can discard any error + let key = self + .api_handler + .key_id_from_request(&req) + .map(|k| format!("(key {k}) ")) + .unwrap_or_default(); + info!("{source} {key}{} {uri}", req.method()); + debug!("{:?}", req); + + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer + .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) + .with_trace_id(gen_trace_id()) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().to_string()), + ]) + .start(&tracer); + + let res = self + .handler_stage2(req) + .with_context(Context::current_with_span(span)) + .await; + + match res { + Ok(x) => { + debug!("{} {:?}", x.status(), x.headers()); + Ok(x) + } + Err(e) => { + let body = e.http_body(&self.region, uri.path()); + let mut http_error_builder = Response::builder().status(e.http_status_code()); + + if let Some(header_map) = http_error_builder.headers_mut() { + e.add_http_headers(header_map) + } + + let http_error = http_error_builder.body(body)?; + + if e.http_status_code().is_server_error() { + warn!("Response: error {}, {}", e.http_status_code(), e); + } else { + info!("Response: error {}, {}", e.http_status_code(), e); + } + Ok(http_error + .map(|body| BoxBody::new(body.map_err(|_: Infallible| unreachable!())))) + } + } + } + + async fn handler_stage2( + &self, + req: Request, + ) -> Result>, A::Error> { + let endpoint = self.api_handler.parse_endpoint(&req)?; + debug!("Endpoint: {}", endpoint.name()); + + let current_context = Context::current(); + let current_span = current_context.span(); + current_span.update_name::(format!( + "{} API {}", + A::API_NAME_DISPLAY, + endpoint.name() + )); + current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); + endpoint.add_span_attributes(current_span); + + let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; + + let res = self + .api_handler + .handle(req, endpoint) + .record_duration(&self.request_duration, &metrics_tags[..]) + .await; + + self.request_counter.add(1, &metrics_tags[..]); + + let status_code = match &res { + Ok(r) => r.status(), + Err(e) => e.http_status_code(), + }; + if status_code.is_client_error() || status_code.is_server_error() { + self.error_counter.add( + 1, + &[ + metrics_tags[0].clone(), + KeyValue::new("status_code", status_code.as_str().to_string()), + ], + ); + } + + res + } +} + +// ==== helper functions ==== + +pub trait Accept: Send + Sync + 'static { + type Stream: AsyncRead + AsyncWrite + Send + Sync + 'static; + fn accept(&self) -> impl Future> + Send; +} + +impl Accept for TcpListener { + type Stream = TcpStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.accept() + .await + .map(|(stream, addr)| (stream, addr.to_string())) + } +} + +pub struct UnixListenerOn(pub UnixListener, pub String); + +impl Accept for UnixListenerOn { + type Stream = UnixStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.0 + .accept() + .await + .map(|(stream, _addr)| (stream, self.1.clone())) + } +} + +pub async fn server_loop( + server_name: String, + listener: A, + handler: H, + mut must_exit: watch::Receiver, +) -> Result<(), GarageError> +where + A: Accept, + H: Fn(Request, String) -> F + Send + Sync + Clone + 'static, + F: Future>, http::Error>> + Send + 'static, + E: Send + Sync + std::error::Error + 'static, +{ + let (conn_in, mut conn_out) = tokio::sync::mpsc::unbounded_channel(); + let connection_collector = tokio::spawn({ + let server_name = server_name.clone(); + async move { + let mut connections = FuturesUnordered::>::new(); + loop { + let collect_next = async { + if connections.is_empty() { + futures::future::pending().await + } else { + connections.next().await + } + }; + tokio::select! { + result = collect_next => { + trace!("{} server: HTTP connection finished: {:?}", server_name, result); + } + new_fut = conn_out.recv() => { + match new_fut { + Some(f) => connections.push(f), + None => break, + } + } + } + } + let deadline = Instant::now() + Duration::from_secs(10); + while !connections.is_empty() { + info!( + "{} server: {} connections still open, deadline in {:.2}s", + server_name, + connections.len(), + (deadline - Instant::now()).as_secs_f32(), + ); + tokio::select! { + conn_res = connections.next() => { + trace!( + "{} server: HTTP connection finished: {:?}", + server_name, + conn_res.unwrap(), + ); + } + _ = sleep_until(deadline) => { + warn!("{} server: exit deadline reached with {} connections still open, killing them now", + server_name, + connections.len()); + for conn in connections.iter() { + conn.abort(); + } + for conn in connections { + assert!(conn.await.unwrap_err().is_cancelled()); + } + break; + } + } + } + } + }); + + while !*must_exit.borrow() { + let (stream, client_addr) = tokio::select! { + acc = listener.accept() => match acc { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue, + Err(e) => return Err(e.into()), + }, + _ = must_exit.changed() => continue, + }; + + let io = TokioIo::new(stream); + + let handler = handler.clone(); + let serve = move |req: Request| handler(req, client_addr.clone()); + + let fut = tokio::task::spawn(async move { + let io = Box::pin(io); + if let Err(e) = http1::Builder::new() + .serve_connection(io, service_fn(serve)) + .await + { + debug!("Error handling HTTP connection: {}", e); + } + }); + conn_in.send(fut)?; + } + + info!("{} server exiting", server_name); + drop(conn_in); + connection_collector.await?; + + Ok(()) +} diff --git a/src/api/helpers.rs b/src/api/common/helpers.rs similarity index 76% rename from src/api/helpers.rs rename to src/api/common/helpers.rs index 642dbc42..6fc4aa13 100644 --- a/src/api/helpers.rs +++ b/src/api/common/helpers.rs @@ -1,7 +1,21 @@ -use hyper::{Body, Request, Response}; -use idna::domain_to_unicode; +use std::convert::Infallible; +use std::sync::Arc; + +use futures::{Stream, StreamExt, TryStreamExt}; + +use http_body_util::{BodyExt, Full as FullBody}; +use hyper::{ + body::{Body, Bytes}, + Request, Response, +}; use serde::{Deserialize, Serialize}; +use garage_model::bucket_table::BucketParams; +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::Uuid; +use garage_util::error::Error as GarageError; + use crate::common_error::{CommonError as Error, *}; /// What kind of authorization is required to perform a given action @@ -17,6 +31,15 @@ pub enum Authorization { Owner, } +/// The values which are known for each request related to a bucket +pub struct ReqCtx { + pub garage: Arc, + pub bucket_id: Uuid, + pub bucket_name: String, + pub bucket_params: BucketParams, + pub api_key: Key, +} + /// Host to bucket /// /// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket", @@ -73,7 +96,7 @@ pub fn authority_to_host(authority: &str) -> Result { authority ))), }; - authority.map(|h| domain_to_unicode(h).0) + authority.map(|h| h.to_ascii_lowercase()) } /// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in @@ -138,18 +161,68 @@ pub fn key_after_prefix(pfx: &str) -> Option { None } -pub async fn parse_json_body Deserialize<'de>>(req: Request) -> Result { - let body = hyper::body::to_bytes(req.into_body()).await?; +// =============== body helpers ================= + +pub type EmptyBody = http_body_util::Empty; +pub type ErrorBody = FullBody; +pub type BoxBody = http_body_util::combinators::BoxBody; + +pub fn string_body(s: String) -> BoxBody { + bytes_body(bytes::Bytes::from(s.into_bytes())) +} +pub fn bytes_body(b: bytes::Bytes) -> BoxBody { + BoxBody::new(FullBody::new(b).map_err(|_: Infallible| unreachable!())) +} +pub fn empty_body() -> BoxBody { + BoxBody::new(http_body_util::Empty::new().map_err(|_: Infallible| unreachable!())) +} +pub fn error_body(s: String) -> ErrorBody { + ErrorBody::from(bytes::Bytes::from(s.into_bytes())) +} + +pub async fn parse_json_body(req: Request) -> Result +where + T: for<'de> Deserialize<'de>, + B: Body, + E: From<::Error> + From, +{ + let body = req.into_body().collect().await?.to_bytes(); let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; Ok(resp) } -pub fn json_ok_response(res: &T) -> Result, Error> { - let resp_json = serde_json::to_string_pretty(res).map_err(garage_util::error::Error::from)?; +pub fn json_ok_response(res: &T) -> Result>, E> +where + E: From, +{ + let resp_json = serde_json::to_string_pretty(res) + .map_err(GarageError::from) + .map_err(Error::from)?; Ok(Response::builder() .status(hyper::StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/json") - .body(Body::from(resp_json))?) + .body(string_body(resp_json)) + .unwrap()) +} + +pub fn body_stream(body: B) -> impl Stream> +where + B: Body, + ::Error: Into, + E: From, +{ + let stream = http_body_util::BodyStream::new(body); + let stream = TryStreamExt::map_err(stream, Into::into); + stream.map(|x| { + x.and_then(|f| { + f.into_data() + .map_err(|_| E::from(Error::bad_request("non-data frame"))) + }) + }) +} + +pub fn is_default(v: &T) -> bool { + *v == T::default() } #[cfg(test)] @@ -289,9 +362,9 @@ mod tests { } #[derive(Serialize)] -pub(crate) struct CustomApiErrorBody { - pub(crate) code: String, - pub(crate) message: String, - pub(crate) region: String, - pub(crate) path: String, +pub struct CustomApiErrorBody { + pub code: String, + pub message: String, + pub region: String, + pub path: String, } diff --git a/src/api/common/lib.rs b/src/api/common/lib.rs new file mode 100644 index 00000000..0e655a53 --- /dev/null +++ b/src/api/common/lib.rs @@ -0,0 +1,12 @@ +//! Crate for serving a S3 compatible API +#[macro_use] +extern crate tracing; + +pub mod common_error; + +pub mod cors; +pub mod encoding; +pub mod generic_server; +pub mod helpers; +pub mod router_macros; +pub mod signature; diff --git a/src/api/router_macros.rs b/src/api/common/router_macros.rs similarity index 96% rename from src/api/router_macros.rs rename to src/api/common/router_macros.rs index 07b5570c..d9fe86db 100644 --- a/src/api/router_macros.rs +++ b/src/api/common/router_macros.rs @@ -1,5 +1,6 @@ /// This macro is used to generate very repetitive match {} blocks in this module /// It is _not_ made to be used anywhere else +#[macro_export] macro_rules! router_match { (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{ // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] } @@ -26,6 +27,7 @@ macro_rules! router_match { $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)* ]) => {{ { + #[allow(unused_parens)] match ($method, $reqpath) { $( (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api { @@ -128,16 +130,11 @@ macro_rules! router_match { } } }; - (@if ($($cond:tt)+) then ($($then:tt)*) else ($($else:tt)*)) => { - $($then)* - }; - (@if () then ($($then:tt)*) else ($($else:tt)*)) => { - $($else)* - }; } /// This macro is used to generate part of the code in this module. It must be called only one, and /// is useless outside of this module. +#[macro_export] macro_rules! generateQueryParameters { ( keywords: [ $($kw_param:expr => $kw_name: ident),* ], @@ -209,7 +206,7 @@ macro_rules! generateQueryParameters { } /// Get an error message in case not all parameters where used when extracting them to - /// build an Enpoint variant + /// build an Endpoint variant fn nonempty_message(&self) -> Option<&str> { if self.keyword.is_some() { Some("Keyword not used") @@ -225,5 +222,5 @@ macro_rules! generateQueryParameters { } } -pub(crate) use generateQueryParameters; -pub(crate) use router_match; +pub use generateQueryParameters; +pub use router_match; diff --git a/src/api/common/signature/body.rs b/src/api/common/signature/body.rs new file mode 100644 index 00000000..96be0d5b --- /dev/null +++ b/src/api/common/signature/body.rs @@ -0,0 +1,135 @@ +use std::sync::Mutex; + +use futures::prelude::*; +use futures::stream::BoxStream; +use http_body_util::{BodyExt, StreamBody}; +use hyper::body::{Bytes, Frame}; +use serde::Deserialize; +use tokio::sync::mpsc; +use tokio::task; + +use super::*; + +use crate::signature::checksum::*; + +pub struct ReqBody { + // why need mutex to be sync?? + pub(crate) stream: Mutex, Error>>>, + pub(crate) checksummer: Checksummer, + pub(crate) expected_checksums: ExpectedChecksums, + pub(crate) trailer_algorithm: Option, +} + +pub type StreamingChecksumReceiver = task::JoinHandle>; + +impl ReqBody { + pub fn add_expected_checksums(&mut self, more: ExpectedChecksums) { + if more.md5.is_some() { + self.expected_checksums.md5 = more.md5; + } + if more.sha256.is_some() { + self.expected_checksums.sha256 = more.sha256; + } + if more.extra.is_some() { + self.expected_checksums.extra = more.extra; + } + self.checksummer.add_expected(&self.expected_checksums); + } + + pub fn add_md5(&mut self) { + self.checksummer.add_md5(); + } + + // ============ non-streaming ============= + + pub async fn json Deserialize<'a>>(self) -> Result { + let body = self.collect().await?; + let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + Ok(resp) + } + + pub async fn collect(self) -> Result { + self.collect_with_checksums().await.map(|(b, _)| b) + } + + pub async fn collect_with_checksums(mut self) -> Result<(Bytes, Checksums), Error> { + let stream: BoxStream<_> = self.stream.into_inner().unwrap(); + let bytes = BodyExt::collect(StreamBody::new(stream)).await?.to_bytes(); + + self.checksummer.update(&bytes); + let checksums = self.checksummer.finalize(); + checksums.verify(&self.expected_checksums)?; + + Ok((bytes, checksums)) + } + + // ============ streaming ============= + + pub fn streaming_with_checksums( + self, + ) -> ( + BoxStream<'static, Result>, + StreamingChecksumReceiver, + ) { + let Self { + stream, + mut checksummer, + mut expected_checksums, + trailer_algorithm, + } = self; + + let (frame_tx, mut frame_rx) = mpsc::channel::>(5); + + let join_checksums = tokio::spawn(async move { + while let Some(frame) = frame_rx.recv().await { + match frame.into_data() { + Ok(data) => { + checksummer = tokio::task::spawn_blocking(move || { + checksummer.update(&data); + checksummer + }) + .await + .unwrap() + } + Err(frame) => { + let trailers = frame.into_trailers().unwrap(); + let algo = trailer_algorithm.unwrap(); + expected_checksums.extra = Some(extract_checksum_value(&trailers, algo)?); + break; + } + } + } + + if trailer_algorithm.is_some() && expected_checksums.extra.is_none() { + return Err(Error::bad_request("trailing checksum was not sent")); + } + + let checksums = checksummer.finalize(); + checksums.verify(&expected_checksums)?; + + Ok(checksums) + }); + + let stream: BoxStream<_> = stream.into_inner().unwrap(); + let stream = stream.filter_map(move |x| { + let frame_tx = frame_tx.clone(); + async move { + match x { + Err(e) => Some(Err(e)), + Ok(frame) => { + if frame.is_data() { + let data = frame.data_ref().unwrap().clone(); + let _ = frame_tx.send(frame).await; + Some(Ok(data)) + } else { + let _ = frame_tx.send(frame).await; + None + } + } + } + } + }); + + (stream.boxed(), join_checksums) + } +} diff --git a/src/api/common/signature/checksum.rs b/src/api/common/signature/checksum.rs new file mode 100644 index 00000000..3c5e7c53 --- /dev/null +++ b/src/api/common/signature/checksum.rs @@ -0,0 +1,323 @@ +use std::convert::{TryFrom, TryInto}; +use std::hash::Hasher; + +use base64::prelude::*; +use crc32c::Crc32cHasher as Crc32c; +use crc32fast::Hasher as Crc32; +use md5::{Digest, Md5}; +use sha1::Sha1; +use sha2::Sha256; + +use http::{HeaderMap, HeaderName, HeaderValue}; + +use garage_util::data::*; + +use super::*; + +pub use garage_model::s3::object_table::{ChecksumAlgorithm, ChecksumValue}; + +pub const CONTENT_MD5: HeaderName = HeaderName::from_static("content-md5"); + +pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-checksum-algorithm"); +pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode"); +pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32"); +pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c"); +pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1"); +pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256"); + +pub type Crc32Checksum = [u8; 4]; +pub type Crc32cChecksum = [u8; 4]; +pub type Md5Checksum = [u8; 16]; +pub type Sha1Checksum = [u8; 20]; +pub type Sha256Checksum = [u8; 32]; + +#[derive(Debug, Default, Clone)] +pub struct ExpectedChecksums { + // base64-encoded md5 (content-md5 header) + pub md5: Option, + // content_sha256 (as a Hash / FixedBytes32) + pub sha256: Option, + // extra x-amz-checksum-* header + pub extra: Option, +} + +pub struct Checksummer { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +#[derive(Default)] +pub struct Checksums { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +impl Checksummer { + pub fn new() -> Self { + Self { + crc32: None, + crc32c: None, + md5: None, + sha1: None, + sha256: None, + } + } + + pub fn init(expected: &ExpectedChecksums, add_md5: bool) -> Self { + let mut ret = Self::new(); + ret.add_expected(expected); + if add_md5 { + ret.add_md5(); + } + ret + } + + pub fn add_md5(&mut self) { + self.md5 = Some(Md5::new()); + } + + pub fn add_expected(&mut self, expected: &ExpectedChecksums) { + if expected.md5.is_some() { + self.md5 = Some(Md5::new()); + } + if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) { + self.sha256 = Some(Sha256::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) { + self.crc32 = Some(Crc32::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) { + self.crc32c = Some(Crc32c::default()); + } + if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) { + self.sha1 = Some(Sha1::new()); + } + } + + pub fn add(mut self, algo: Option) -> Self { + match algo { + Some(ChecksumAlgorithm::Crc32) => { + self.crc32 = Some(Crc32::new()); + } + Some(ChecksumAlgorithm::Crc32c) => { + self.crc32c = Some(Crc32c::default()); + } + Some(ChecksumAlgorithm::Sha1) => { + self.sha1 = Some(Sha1::new()); + } + Some(ChecksumAlgorithm::Sha256) => { + self.sha256 = Some(Sha256::new()); + } + None => (), + } + self + } + + pub fn update(&mut self, bytes: &[u8]) { + if let Some(crc32) = &mut self.crc32 { + crc32.update(bytes); + } + if let Some(crc32c) = &mut self.crc32c { + crc32c.write(bytes); + } + if let Some(md5) = &mut self.md5 { + md5.update(bytes); + } + if let Some(sha1) = &mut self.sha1 { + sha1.update(bytes); + } + if let Some(sha256) = &mut self.sha256 { + sha256.update(bytes); + } + } + + pub fn finalize(self) -> Checksums { + Checksums { + crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())), + crc32c: self + .crc32c + .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())), + md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()), + sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()), + sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()), + } + } +} + +impl Checksums { + pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> { + if let Some(expected_md5) = &expected.md5 { + match self.md5 { + Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (), + _ => { + return Err(Error::InvalidDigest( + "MD5 checksum verification failed (from content-md5)".into(), + )) + } + } + } + if let Some(expected_sha256) = &expected.sha256 { + match self.sha256 { + Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (), + _ => { + return Err(Error::InvalidDigest( + "SHA256 checksum verification failed (from x-amz-content-sha256)".into(), + )) + } + } + } + if let Some(extra) = expected.extra { + let algo = extra.algorithm(); + if self.extract(Some(algo)) != Some(extra) { + return Err(Error::InvalidDigest(format!( + "Failed to validate checksum for algorithm {:?}", + algo + ))); + } + } + Ok(()) + } + + pub fn extract(&self, algo: Option) -> Option { + match algo { + None => None, + Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())), + Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())), + Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())), + Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())), + } + } +} + +// ---- + +pub fn parse_checksum_algorithm(algo: &str) -> Result { + match algo { + "CRC32" => Ok(ChecksumAlgorithm::Crc32), + "CRC32C" => Ok(ChecksumAlgorithm::Crc32c), + "SHA1" => Ok(ChecksumAlgorithm::Sha1), + "SHA256" => Ok(ChecksumAlgorithm::Sha256), + _ => Err(Error::bad_request("invalid checksum algorithm")), + } +} + +/// Extract the value of the x-amz-checksum-algorithm header +pub fn request_checksum_algorithm( + headers: &HeaderMap, +) -> Result, Error> { + match headers.get(X_AMZ_CHECKSUM_ALGORITHM) { + None => Ok(None), + Some(x) => parse_checksum_algorithm(x.to_str()?).map(Some), + } +} + +pub fn request_trailer_checksum_algorithm( + headers: &HeaderMap, +) -> Result, Error> { + match headers.get(X_AMZ_TRAILER).map(|x| x.to_str()).transpose()? { + None => Ok(None), + Some(x) if x == X_AMZ_CHECKSUM_CRC32 => Ok(Some(ChecksumAlgorithm::Crc32)), + Some(x) if x == X_AMZ_CHECKSUM_CRC32C => Ok(Some(ChecksumAlgorithm::Crc32c)), + Some(x) if x == X_AMZ_CHECKSUM_SHA1 => Ok(Some(ChecksumAlgorithm::Sha1)), + Some(x) if x == X_AMZ_CHECKSUM_SHA256 => Ok(Some(ChecksumAlgorithm::Sha256)), + _ => Err(Error::bad_request("invalid checksum algorithm")), + } +} + +/// Extract the value of any of the x-amz-checksum-* headers +pub fn request_checksum_value( + headers: &HeaderMap, +) -> Result, Error> { + let mut ret = vec![]; + + if headers.contains_key(X_AMZ_CHECKSUM_CRC32) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_CRC32C) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32c)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_SHA1) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha1)?); + } + if headers.contains_key(X_AMZ_CHECKSUM_SHA256) { + ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha256)?); + } + + if ret.len() > 1 { + return Err(Error::bad_request( + "multiple x-amz-checksum-* headers given", + )); + } + Ok(ret.pop()) +} + +/// Checks for the presence of x-amz-checksum-algorithm +/// if so extract the corresponding x-amz-checksum-* value +pub fn extract_checksum_value( + headers: &HeaderMap, + algo: ChecksumAlgorithm, +) -> Result { + match algo { + ChecksumAlgorithm::Crc32 => { + let crc32 = headers + .get(X_AMZ_CHECKSUM_CRC32) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-crc32 header")?; + Ok(ChecksumValue::Crc32(crc32)) + } + ChecksumAlgorithm::Crc32c => { + let crc32c = headers + .get(X_AMZ_CHECKSUM_CRC32C) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-crc32c header")?; + Ok(ChecksumValue::Crc32c(crc32c)) + } + ChecksumAlgorithm::Sha1 => { + let sha1 = headers + .get(X_AMZ_CHECKSUM_SHA1) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-sha1 header")?; + Ok(ChecksumValue::Sha1(sha1)) + } + ChecksumAlgorithm::Sha256 => { + let sha256 = headers + .get(X_AMZ_CHECKSUM_SHA256) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-sha256 header")?; + Ok(ChecksumValue::Sha256(sha256)) + } + } +} + +pub fn add_checksum_response_headers( + checksum: &Option, + mut resp: http::response::Builder, +) -> http::response::Builder { + match checksum { + Some(ChecksumValue::Crc32(crc32)) => { + resp = resp.header(X_AMZ_CHECKSUM_CRC32, BASE64_STANDARD.encode(&crc32)); + } + Some(ChecksumValue::Crc32c(crc32c)) => { + resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c)); + } + Some(ChecksumValue::Sha1(sha1)) => { + resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1)); + } + Some(ChecksumValue::Sha256(sha256)) => { + resp = resp.header(X_AMZ_CHECKSUM_SHA256, BASE64_STANDARD.encode(&sha256)); + } + None => (), + } + resp +} diff --git a/src/api/signature/error.rs b/src/api/common/signature/error.rs similarity index 60% rename from src/api/signature/error.rs rename to src/api/common/signature/error.rs index f0d7c816..a1b353e1 100644 --- a/src/api/signature/error.rs +++ b/src/api/common/signature/error.rs @@ -1,4 +1,4 @@ -use err_derive::Error; +use thiserror::Error; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; @@ -6,22 +6,22 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error Common(CommonError), /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), } impl From for Error diff --git a/src/api/common/signature/mod.rs b/src/api/common/signature/mod.rs new file mode 100644 index 00000000..50fbd304 --- /dev/null +++ b/src/api/common/signature/mod.rs @@ -0,0 +1,118 @@ +use chrono::{DateTime, Utc}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use hyper::header::HeaderName; +use hyper::{body::Incoming as IncomingBody, Request}; + +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::{sha256sum, Hash}; + +use error::*; + +pub mod body; +pub mod checksum; +pub mod error; +pub mod payload; +pub mod streaming; + +pub const SHORT_DATE: &str = "%Y%m%d"; +pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; + +// ---- Constants used in AWSv4 signatures ---- + +pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm"); +pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential"); +pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date"); +pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires"); +pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders"); +pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature"); +pub const X_AMZ_CONTENT_SHA256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); +pub const X_AMZ_TRAILER: HeaderName = HeaderName::from_static("x-amz-trailer"); + +/// Result of `sha256("")` +pub(crate) const EMPTY_STRING_HEX_DIGEST: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +// Signature calculation algorithm +pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; +type HmacSha256 = Hmac; + +// Possible values for x-amz-content-sha256, in addition to the actual sha256 +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const STREAMING_UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; +pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + +// Used in the computation of StringToSign +pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD"; + +// ---- enums to describe stuff going on in signature calculation ---- + +#[derive(Debug)] +pub enum ContentSha256Header { + UnsignedPayload, + Sha256Checksum(Hash), + StreamingPayload { trailer: bool, signed: bool }, +} + +// ---- top-level functions ---- + +pub struct VerifiedRequest { + pub request: Request, + pub access_key: Key, + pub content_sha256_header: ContentSha256Header, +} + +pub async fn verify_request( + garage: &Garage, + mut req: Request, + service: &'static str, +) -> Result { + let checked_signature = payload::check_payload_signature(&garage, &mut req, service).await?; + + let request = streaming::parse_streaming_body( + req, + &checked_signature, + &garage.config.s3_api.s3_region, + service, + )?; + + let access_key = checked_signature + .key + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; + + Ok(VerifiedRequest { + request, + access_key, + content_sha256_header: checked_signature.content_sha256_header, + }) +} + +pub fn signing_hmac( + datetime: &DateTime, + secret_key: &str, + region: &str, + service: &str, +) -> Result { + let secret = String::from("AWS4") + secret_key; + let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; + date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); + let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; + region_hmac.update(region.as_bytes()); + let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; + service_hmac.update(service.as_bytes()); + let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; + signing_hmac.update(b"aws4_request"); + let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; + Ok(hmac) +} + +pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { + format!( + "{}/{}/{}/aws4_request", + datetime.format(SHORT_DATE), + region, + service + ) +} diff --git a/src/api/common/signature/payload.rs b/src/api/common/signature/payload.rs new file mode 100644 index 00000000..3939da19 --- /dev/null +++ b/src/api/common/signature/payload.rs @@ -0,0 +1,584 @@ +use std::collections::HashMap; +use std::convert::TryFrom; + +use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; +use hmac::Mac; +use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST}; +use hyper::{body::Incoming as IncomingBody, Method, Request}; +use sha2::{Digest, Sha256}; + +use garage_table::*; +use garage_util::data::Hash; + +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use super::*; + +use crate::encoding::uri_encode; + +pub type QueryMap = HeaderMap; +pub struct QueryValue { + /// Original key with potential uppercase characters, + /// for use in signature calculation + key: String, + value: String, +} + +#[derive(Debug)] +pub struct CheckedSignature { + pub key: Option, + pub content_sha256_header: ContentSha256Header, + pub signature_header: Option, +} + +pub async fn check_payload_signature( + garage: &Garage, + request: &mut Request, + service: &'static str, +) -> Result { + let query = parse_query_map(request.uri())?; + + if query.contains_key(&X_AMZ_ALGORITHM) { + // We check for presigned-URL-style authentication first, because + // the browser or something else could inject an Authorization header + // that is totally unrelated to AWS signatures. + check_presigned_signature(garage, service, request, query).await + } else if request.headers().contains_key(AUTHORIZATION) { + check_standard_signature(garage, service, request, query).await + } else { + // Unsigned (anonymous) request + let content_sha256 = request + .headers() + .get(X_AMZ_CONTENT_SHA256) + .map(|x| x.to_str()) + .transpose()?; + Ok(CheckedSignature { + key: None, + content_sha256_header: parse_x_amz_content_sha256(content_sha256)?, + signature_header: None, + }) + } +} + +fn parse_x_amz_content_sha256(header: Option<&str>) -> Result { + let header = match header { + Some(x) => x, + None => return Ok(ContentSha256Header::UnsignedPayload), + }; + if header == UNSIGNED_PAYLOAD { + Ok(ContentSha256Header::UnsignedPayload) + } else if let Some(rest) = header.strip_prefix("STREAMING-") { + let (trailer, algo) = if let Some(rest2) = rest.strip_suffix("-TRAILER") { + (true, rest2) + } else { + (false, rest) + }; + let signed = match algo { + AWS4_HMAC_SHA256_PAYLOAD => true, + UNSIGNED_PAYLOAD => false, + _ => { + return Err(Error::bad_request( + "invalid or unsupported x-amz-content-sha256", + )) + } + }; + Ok(ContentSha256Header::StreamingPayload { trailer, signed }) + } else { + let sha256 = hex::decode(header) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid content sha256 hash")?; + Ok(ContentSha256Header::Sha256Checksum(sha256)) + } +} + +async fn check_standard_signature( + garage: &Garage, + service: &'static str, + request: &Request, + query: QueryMap, +) -> Result { + let authorization = Authorization::parse_header(request.headers())?; + + // Verify that all necessary request headers are included in signed_headers + // The following must be included for all signatures: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request (except x-amz-content-sha256) + // AWS also indicates that the Content-Type header should be signed if + // it is used, but Minio client doesn't sign it so we don't check it for compatibility. + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request:\n{}", canonical_request); + trace!("string to sign:\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + let content_sha256_header = parse_x_amz_content_sha256(Some(&authorization.content_sha256))?; + + Ok(CheckedSignature { + key: Some(key), + content_sha256_header, + signature_header: Some(authorization.signature), + }) +} + +async fn check_presigned_signature( + garage: &Garage, + service: &'static str, + request: &mut Request, + mut query: QueryMap, +) -> Result { + let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap(); + let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; + + // Verify that all necessary request headers are included in signed_headers + // For AWSv4 pre-signed URLs, the following must be included: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request (except x-amz-content-sha256) + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + // The X-Amz-Signature value is passed as a query parameter, + // but the signature cannot be computed from a string that contains itself. + // AWS specifies that all query params except X-Amz-Signature are included + // in the canonical request. + query.remove(&X_AMZ_SIGNATURE); + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request (presigned url):\n{}", canonical_request); + trace!("string to sign (presigned url):\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + // In the page on presigned URLs, AWS specifies that if a signed query + // parameter and a signed header of the same name have different values, + // then an InvalidRequest error is raised. + let headers_mut = request.headers_mut(); + for (name, value) in query.iter() { + if let Some(existing) = headers_mut.get(name) { + if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { + return Err(Error::bad_request(format!( + "Conflicting values for `{}` in query parameters and request headers", + name + ))); + } + } + if name.as_str().starts_with("x-amz-") { + // Query parameters that start by x-amz- are actually intended to stand in for + // headers that can't be added at the time the request is made. + // What we do is just add them to the Request object as regular headers, + // that will be handled downstream as if they were included like in a normal request. + // (Here we allow such query parameters to override headers with the same name + // that are not signed, however there is not much reason that this would happen) + headers_mut.insert( + name, + HeaderValue::from_bytes(value.value.as_bytes()) + .ok_or_bad_request("invalid query parameter value")?, + ); + } + } + + // Presigned URLs always use UNSIGNED-PAYLOAD, + // so there is no sha256 hash to return. + Ok(CheckedSignature { + key: Some(key), + content_sha256_header: ContentSha256Header::UnsignedPayload, + signature_header: Some(authorization.signature), + }) +} + +pub fn parse_query_map(uri: &http::uri::Uri) -> Result { + let mut query = QueryMap::with_capacity(0); + if let Some(query_str) = uri.query() { + let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); + for (key, val) in query_pairs { + let name = + HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?; + + let value = QueryValue { + key: key.to_string(), + value: val.into_owned(), + }; + + if query.insert(name, value).is_some() { + return Err(Error::bad_request(format!( + "duplicate query parameter: `{}`", + key + ))); + } + } + } + Ok(query) +} + +fn parse_credential(cred: &str) -> Result<(String, String), Error> { + let first_slash = cred + .find('/') + .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; + let (key_id, scope) = cred.split_at(first_slash); + Ok(( + key_id.to_string(), + scope.trim_start_matches('/').to_string(), + )) +} + +fn split_signed_headers(authorization: &Authorization) -> Result, Error> { + let mut signed_headers = authorization + .signed_headers + .split(';') + .map(HeaderName::try_from) + .collect::, _>>() + .ok_or_bad_request("invalid header name")?; + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + Ok(signed_headers) +} + +fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> { + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } + for (name, _) in headers.iter() { + // Enforce signature of all x-amz-* headers, except x-amz-content-sh256 + // because it is included in the canonical request in all cases + if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } + } + Ok(()) +} + +pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { + let mut hasher = Sha256::default(); + hasher.update(canonical_req.as_bytes()); + [ + AWS4_HMAC_SHA256, + &datetime.format(LONG_DATETIME).to_string(), + scope_string, + &hex::encode(hasher.finalize().as_slice()), + ] + .join("\n") +} + +pub fn canonical_request( + service: &'static str, + method: &Method, + canonical_uri: &str, + query: &QueryMap, + headers: &HeaderMap, + signed_headers: &[HeaderName], + content_sha256: &str, +) -> Result { + // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded + // a second time when building the canonical request, as specified in this documentation page: + // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html + // However this documentation page is for a specific service ("roles anywhere"), and + // in the S3 service we know for a fact that there is no double-urlencoding, because all of + // the tests we made with external software work without it. + // + // The theory is that double-urlencoding occurs for all services except S3, + // which is what is implemented in rusoto_signature: + // -> https://docs.rs/rusoto_signature/latest/src/rusoto_signature/signature.rs.html#464 + // + // Digging into the code of the official AWS Rust SDK, we learn that double-URI-encoding can + // be set or unset on a per-request basis (the signature crates, aws-sigv4 and aws-sig-auth, + // are agnostic to this). Grepping the codebase confirms that S3 is the only API for which + // double_uri_encode is set to false, meaning it is true (its default value) for all other + // AWS services. We will therefore implement this behavior in Garage as well. + // + // Note that this documentation page, which is touted as the "authoritative reference" on + // AWSv4 signatures, makes no mention of either single- or double-urlencoding: + // -> https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html + // This page of the S3 documentation does also not mention anything specific: + // -> https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // + // Note that there is also the issue of path normalization, which I hope is unrelated to the + // one of URI-encoding. At least in aws-sigv4 both parameters can be set independently, + // and rusoto_signature does not seem to do any effective path normalization, even though + // it mentions it in the comments (same link to the source code as above). + // We make the explicit choice of NOT normalizing paths in the K2V API because doing so + // would make non-normalized paths invalid K2V partition keys, and we don't want that. + let canonical_uri: std::borrow::Cow = if service != "s3" { + uri_encode(canonical_uri, false).into() + } else { + canonical_uri.into() + }; + + // Canonical query string from passed HeaderMap + let canonical_query_string = { + let mut items = Vec::with_capacity(query.len()); + for (_, QueryValue { key, value }) in query.iter() { + items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); + } + items.sort(); + items.join("&") + }; + + // Canonical header string calculated from signed headers + let canonical_header_string = signed_headers + .iter() + .map(|name| { + let value = headers + .get(name) + .ok_or_bad_request(format!("signed header `{}` is not present", name))?; + let value = std::str::from_utf8(value.as_bytes())?; + Ok(format!("{}:{}", name.as_str(), value.trim())) + }) + .collect::, Error>>()? + .join("\n"); + let signed_headers = signed_headers.join(";"); + + let list = [ + method.as_str(), + &canonical_uri, + &canonical_query_string, + &canonical_header_string, + "", + &signed_headers, + content_sha256, + ]; + Ok(list.join("\n")) +} + +pub fn parse_date(date: &str) -> Result, Error> { + let date: NaiveDateTime = + NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; + Ok(Utc.from_utc_datetime(&date)) +} + +pub async fn verify_v4( + garage: &Garage, + service: &str, + auth: &Authorization, + payload: &[u8], +) -> Result { + let scope_expected = compute_scope(&auth.date, &garage.config.s3_api.s3_region, service); + if auth.scope != scope_expected { + return Err(Error::AuthorizationHeaderMalformed(auth.scope.to_string())); + } + + let key = garage + .key_table + .get(&EmptyKey, &auth.key_id) + .await? + .filter(|k| !k.state.is_deleted()) + .ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?; + let key_p = key.params().unwrap(); + + let mut hmac = signing_hmac( + &auth.date, + &key_p.secret_key, + &garage.config.s3_api.s3_region, + service, + ) + .ok_or_internal_error("Unable to build signing HMAC")?; + hmac.update(payload); + let signature = + hex::decode(&auth.signature).map_err(|_| Error::forbidden("Invalid signature"))?; + if hmac.verify_slice(&signature).is_err() { + return Err(Error::forbidden("Invalid signature")); + } + + Ok(key) +} + +// ============ Authorization header, or X-Amz-* query params ========= + +pub struct Authorization { + pub key_id: String, + scope: String, + signed_headers: String, + signature: String, + content_sha256: String, + date: DateTime, +} + +impl Authorization { + pub fn parse_header(headers: &HeaderMap) -> Result { + let authorization = headers + .get(AUTHORIZATION) + .ok_or_bad_request("Missing authorization header")? + .to_str()?; + + let (auth_kind, rest) = authorization + .split_once(' ') + .ok_or_bad_request("Authorization field to short")?; + + if auth_kind != AWS4_HMAC_SHA256 { + return Err(Error::bad_request("Unsupported authorization method")); + } + + let mut auth_params = HashMap::new(); + for auth_part in rest.split(',') { + let auth_part = auth_part.trim(); + let eq = auth_part + .find('=') + .ok_or_bad_request("Field without value in authorization header")?; + let (key, value) = auth_part.split_at(eq); + auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); + } + + let cred = auth_params + .get("Credential") + .ok_or_bad_request("Could not find Credential in Authorization field")?; + let signed_headers = auth_params + .get("SignedHeaders") + .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? + .to_string(); + let signature = auth_params + .get("Signature") + .ok_or_bad_request("Could not find Signature in Authorization field")? + .to_string(); + + let content_sha256 = headers + .get(X_AMZ_CONTENT_SHA256) + .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; + + let date = headers + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(cred)?; + let auth = Authorization { + key_id, + scope, + signed_headers, + signature, + content_sha256: content_sha256.to_str()?.to_string(), + date, + }; + Ok(auth) + } + + fn parse_presigned(algorithm: &str, query: &QueryMap) -> Result { + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let cred = query + .get(&X_AMZ_CREDENTIAL) + .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; + let signed_headers = query + .get(&X_AMZ_SIGNEDHEADERS) + .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; + let signature = query + .get(&X_AMZ_SIGNATURE) + .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; + + let duration = query + .get(&X_AMZ_EXPIRES) + .ok_or_bad_request("X-Amz-Expires not found in query parameters")? + .value + .parse() + .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; + + if duration > 7 * 24 * 3600 { + return Err(Error::bad_request( + "X-Amz-Expires may not exceed a week".to_string(), + )); + } + + let date = query + .get(&X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")?; + let date = parse_date(&date.value)?; + + if Utc::now() - date > Duration::seconds(duration) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(&cred.value)?; + Ok(Authorization { + key_id, + scope, + signed_headers: signed_headers.value.clone(), + signature: signature.value.clone(), + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }) + } + + pub fn parse_form(params: &HeaderMap) -> Result { + let algorithm = params + .get(X_AMZ_ALGORITHM) + .ok_or_bad_request("Missing X-Amz-Algorithm header")? + .to_str()?; + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let credential = params + .get(X_AMZ_CREDENTIAL) + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? + .to_str()?; + let signature = params + .get(X_AMZ_SIGNATURE) + .ok_or_bad_request("No signature was provided")? + .to_str()? + .to_string(); + let date = params + .get(X_AMZ_DATE) + .ok_or_bad_request("No date was provided")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(credential)?; + let auth = Authorization { + key_id, + scope, + signed_headers: "".to_string(), + signature, + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }; + Ok(auth) + } +} diff --git a/src/api/common/signature/streaming.rs b/src/api/common/signature/streaming.rs new file mode 100644 index 00000000..64362727 --- /dev/null +++ b/src/api/common/signature/streaming.rs @@ -0,0 +1,618 @@ +use std::pin::Pin; +use std::sync::Mutex; + +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; +use futures::prelude::*; +use futures::task; +use hmac::Mac; +use http::header::{HeaderMap, HeaderValue, CONTENT_ENCODING}; +use hyper::body::{Bytes, Frame, Incoming as IncomingBody}; +use hyper::Request; + +use garage_util::data::Hash; + +use super::*; + +use crate::helpers::body_stream; +use crate::signature::checksum::*; +use crate::signature::payload::CheckedSignature; + +pub use crate::signature::body::ReqBody; + +pub fn parse_streaming_body( + mut req: Request, + checked_signature: &CheckedSignature, + region: &str, + service: &str, +) -> Result, Error> { + debug!( + "Content signature mode: {:?}", + checked_signature.content_sha256_header + ); + + match checked_signature.content_sha256_header { + ContentSha256Header::StreamingPayload { signed, trailer } => { + // Sanity checks + if !signed && !trailer { + return Err(Error::bad_request( + "STREAMING-UNSIGNED-PAYLOAD without trailer is not a valid combination", + )); + } + + // Remove the aws-chunked component in the content-encoding: header + // Note: this header is not properly sent by minio client, so don't fail + // if it is absent from the request. + if let Some(content_encoding) = req.headers_mut().remove(CONTENT_ENCODING) { + if let Some(rest) = content_encoding.as_bytes().strip_prefix(b"aws-chunked,") { + req.headers_mut() + .insert(CONTENT_ENCODING, HeaderValue::from_bytes(rest).unwrap()); + } else if content_encoding != "aws-chunked" { + return Err(Error::bad_request( + "content-encoding does not contain aws-chunked for STREAMING-*-PAYLOAD", + )); + } + } + + // If trailer header is announced, add the calculation of the requested checksum + let mut checksummer = Checksummer::init(&Default::default(), false); + let trailer_algorithm = if trailer { + let algo = Some( + request_trailer_checksum_algorithm(req.headers())? + .ok_or_bad_request("Missing x-amz-trailer header")?, + ); + checksummer = checksummer.add(algo); + algo + } else { + None + }; + + // For signed variants, determine signing parameters + let sign_params = if signed { + let signature = checked_signature + .signature_header + .clone() + .ok_or_bad_request("No signature provided")?; + let signature = hex::decode(signature) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid signature")?; + + let secret_key = checked_signature + .key + .as_ref() + .ok_or_bad_request("Cannot sign streaming payload without signing key")? + .state + .as_option() + .ok_or_internal_error("Deleted key state")? + .secret_key + .to_string(); + + let date = req + .headers() + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) + .ok_or_bad_request("Invalid date")?; + let date: DateTime = Utc.from_utc_datetime(&date); + + let scope = compute_scope(&date, region, service); + let signing_hmac = + crate::signature::signing_hmac(&date, &secret_key, region, service) + .ok_or_internal_error("Unable to build signing HMAC")?; + + Some(SignParams { + datetime: date, + scope, + signing_hmac, + previous_signature: signature, + }) + } else { + None + }; + + Ok(req.map(move |body| { + let stream = body_stream::<_, Error>(body); + + let signed_payload_stream = + StreamingPayloadStream::new(stream, sign_params, trailer).map_err(Error::from); + ReqBody { + stream: Mutex::new(signed_payload_stream.boxed()), + checksummer, + expected_checksums: Default::default(), + trailer_algorithm, + } + })) + } + _ => Ok(req.map(|body| { + let expected_checksums = ExpectedChecksums { + sha256: match &checked_signature.content_sha256_header { + ContentSha256Header::Sha256Checksum(sha256) => Some(*sha256), + _ => None, + }, + ..Default::default() + }; + let checksummer = Checksummer::init(&expected_checksums, false); + + let stream = http_body_util::BodyStream::new(body).map_err(Error::from); + ReqBody { + stream: Mutex::new(stream.boxed()), + checksummer, + expected_checksums, + trailer_algorithm: None, + } + })), + } +} + +fn compute_streaming_payload_signature( + signing_hmac: &HmacSha256, + date: DateTime, + scope: &str, + previous_signature: Hash, + content_sha256: Hash, +) -> Result { + let string_to_sign = [ + AWS4_HMAC_SHA256_PAYLOAD, + &date.format(LONG_DATETIME).to_string(), + scope, + &hex::encode(previous_signature), + EMPTY_STRING_HEX_DIGEST, + &hex::encode(content_sha256), + ] + .join("\n"); + + let mut hmac = signing_hmac.clone(); + hmac.update(string_to_sign.as_bytes()); + + Hash::try_from(&hmac.finalize().into_bytes()) + .ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into())) +} + +fn compute_streaming_trailer_signature( + signing_hmac: &HmacSha256, + date: DateTime, + scope: &str, + previous_signature: Hash, + trailer_sha256: Hash, +) -> Result { + let string_to_sign = [ + AWS4_HMAC_SHA256_PAYLOAD, + &date.format(LONG_DATETIME).to_string(), + scope, + &hex::encode(previous_signature), + &hex::encode(trailer_sha256), + ] + .join("\n"); + + let mut hmac = signing_hmac.clone(); + hmac.update(string_to_sign.as_bytes()); + + Hash::try_from(&hmac.finalize().into_bytes()) + .ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into())) +} + +mod payload { + use http::{HeaderName, HeaderValue}; + + use garage_util::data::Hash; + + use nom::bytes::streaming::{tag, take_while}; + use nom::character::streaming::hex_digit1; + use nom::combinator::{map_res, opt}; + use nom::number::streaming::hex_u32; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(|e| e.map(Error::Parser))? + }; + } + + pub enum Error { + Parser(nom::error::Error), + BadSignature, + } + + impl Error { + pub fn description(&self) -> &str { + match *self { + Error::Parser(ref e) => e.code.description(), + Error::BadSignature => "Bad signature", + } + } + } + + #[derive(Debug, Clone)] + pub struct ChunkHeader { + pub size: usize, + pub signature: Option, + } + + impl ChunkHeader { + pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, size) = try_parse!(hex_u32(input)); + let (input, _) = try_parse!(tag(";")(input)); + + let (input, _) = try_parse!(tag("chunk-signature=")(input)); + let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); + let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; + + let (input, _) = try_parse!(tag("\r\n")(input)); + + let header = ChunkHeader { + size: size as usize, + signature: Some(signature), + }; + + Ok((input, header)) + } + + pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, size) = try_parse!(hex_u32(input)); + let (input, _) = try_parse!(tag("\r\n")(input)); + + let header = ChunkHeader { + size: size as usize, + signature: None, + }; + + Ok((input, header)) + } + } + + #[derive(Debug, Clone)] + pub struct TrailerChunk { + pub header_name: HeaderName, + pub header_value: HeaderValue, + pub signature: Option, + } + + impl TrailerChunk { + fn parse_content(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, header_name) = try_parse!(map_res( + take_while(|c: u8| c.is_ascii_alphanumeric() || c == b'-'), + HeaderName::from_bytes + )(input)); + let (input, _) = try_parse!(tag(b":")(input)); + let (input, header_value) = try_parse!(map_res( + take_while(|c: u8| c.is_ascii_alphanumeric() || b"+/=".contains(&c)), + HeaderValue::from_bytes + )(input)); + + // Possible '\n' after the header value, depends on clients + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + let (input, _) = try_parse!(opt(tag(b"\n"))(input)); + + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok(( + input, + TrailerChunk { + header_name, + header_value, + signature: None, + }, + )) + } + pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, trailer) = Self::parse_content(input)?; + + let (input, _) = try_parse!(tag(b"x-amz-trailer-signature:")(input)); + let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); + let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok(( + input, + TrailerChunk { + signature: Some(signature), + ..trailer + }, + )) + } + pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + let (input, trailer) = Self::parse_content(input)?; + let (input, _) = try_parse!(tag(b"\r\n")(input)); + + Ok((input, trailer)) + } + } +} + +#[derive(Debug)] +pub enum StreamingPayloadError { + Stream(Error), + InvalidSignature, + Message(String), +} + +impl StreamingPayloadError { + fn message(msg: &str) -> Self { + StreamingPayloadError::Message(msg.into()) + } +} + +impl From for Error { + fn from(err: StreamingPayloadError) -> Self { + match err { + StreamingPayloadError::Stream(e) => e, + StreamingPayloadError::InvalidSignature => { + Error::bad_request("Invalid payload signature") + } + StreamingPayloadError::Message(e) => { + Error::bad_request(format!("Chunk format error: {}", e)) + } + } + } +} + +impl From> for StreamingPayloadError { + fn from(err: payload::Error) -> Self { + Self::message(err.description()) + } +} + +impl From> for StreamingPayloadError { + fn from(err: nom::error::Error) -> Self { + Self::message(err.code.description()) + } +} + +enum StreamingPayloadChunk { + Chunk { + header: payload::ChunkHeader, + data: Bytes, + }, + Trailer(payload::TrailerChunk), +} + +struct SignParams { + datetime: DateTime, + scope: String, + signing_hmac: HmacSha256, + previous_signature: Hash, +} + +#[pin_project::pin_project] +pub struct StreamingPayloadStream +where + S: Stream>, +{ + #[pin] + stream: S, + buf: bytes::BytesMut, + signing: Option, + has_trailer: bool, + done: bool, +} + +impl StreamingPayloadStream +where + S: Stream>, +{ + fn new(stream: S, signing: Option, has_trailer: bool) -> Self { + Self { + stream, + buf: bytes::BytesMut::new(), + signing, + has_trailer, + done: false, + } + } + + fn parse_next( + input: &[u8], + is_signed: bool, + has_trailer: bool, + ) -> nom::IResult<&[u8], StreamingPayloadChunk, StreamingPayloadError> { + use nom::bytes::streaming::{tag, take}; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(nom::Err::convert)? + }; + } + + let (input, header) = if is_signed { + try_parse!(payload::ChunkHeader::parse_signed(input)) + } else { + try_parse!(payload::ChunkHeader::parse_unsigned(input)) + }; + + // 0-sized chunk is the last + if header.size == 0 { + if has_trailer { + let (input, trailer) = if is_signed { + try_parse!(payload::TrailerChunk::parse_signed(input)) + } else { + try_parse!(payload::TrailerChunk::parse_unsigned(input)) + }; + return Ok((input, StreamingPayloadChunk::Trailer(trailer))); + } else { + return Ok(( + input, + StreamingPayloadChunk::Chunk { + header, + data: Bytes::new(), + }, + )); + } + } + + let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); + let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); + + let data = Bytes::from(data.to_vec()); + + Ok((input, StreamingPayloadChunk::Chunk { header, data })) + } +} + +impl Stream for StreamingPayloadStream +where + S: Stream> + Unpin, +{ + type Item = Result, StreamingPayloadError>; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll> { + use std::task::Poll; + + let mut this = self.project(); + + if *this.done { + return Poll::Ready(None); + } + + loop { + let (input, payload) = + match Self::parse_next(this.buf, this.signing.is_some(), *this.has_trailer) { + Ok(res) => res, + Err(nom::Err::Incomplete(_)) => { + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + continue; + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(StreamingPayloadError::Stream(e)))) + } + None => { + return Poll::Ready(Some(Err(StreamingPayloadError::message( + "Unexpected EOF", + )))); + } + } + } + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + return Poll::Ready(Some(Err(e))) + } + }; + + match payload { + StreamingPayloadChunk::Chunk { data, header } => { + if let Some(signing) = this.signing.as_mut() { + let data_sha256sum = sha256sum(&data); + + let expected_signature = compute_streaming_payload_signature( + &signing.signing_hmac, + signing.datetime, + &signing.scope, + signing.previous_signature, + data_sha256sum, + )?; + + if header.signature.unwrap() != expected_signature { + return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature))); + } + + signing.previous_signature = header.signature.unwrap(); + } + + *this.buf = input.into(); + + // 0-sized chunk is the last + if data.is_empty() { + // if there was a trailer, it would have been returned by the parser + assert!(!*this.has_trailer); + *this.done = true; + return Poll::Ready(None); + } + + return Poll::Ready(Some(Ok(Frame::data(data)))); + } + StreamingPayloadChunk::Trailer(trailer) => { + trace!( + "In StreamingPayloadStream::poll_next: got trailer {:?}", + trailer + ); + + if let Some(signing) = this.signing.as_mut() { + let data = [ + trailer.header_name.as_ref(), + &b":"[..], + trailer.header_value.as_ref(), + &b"\n"[..], + ] + .concat(); + let trailer_sha256sum = sha256sum(&data); + + let expected_signature = compute_streaming_trailer_signature( + &signing.signing_hmac, + signing.datetime, + &signing.scope, + signing.previous_signature, + trailer_sha256sum, + )?; + + if trailer.signature.unwrap() != expected_signature { + return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature))); + } + } + + *this.buf = input.into(); + *this.done = true; + + let mut trailers_map = HeaderMap::new(); + trailers_map.insert(trailer.header_name, trailer.header_value); + + return Poll::Ready(Some(Ok(Frame::trailers(trailers_map)))); + } + } + } + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} + +#[cfg(test)] +mod tests { + use futures::prelude::*; + + use super::{SignParams, StreamingPayloadError, StreamingPayloadStream}; + + #[tokio::test] + async fn test_interrupted_signed_payload_stream() { + use chrono::{DateTime, Utc}; + + use garage_util::data::Hash; + + let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 + .unwrap() + .with_timezone(&Utc); + let secret_key = "test"; + let region = "test"; + let scope = crate::signature::compute_scope(&datetime, region, "s3"); + let signing_hmac = + crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); + + let data: &[&[u8]] = &[b"1"]; + let body = futures::stream::iter(data.iter().map(|block| Ok(block.to_vec().into()))); + + let seed_signature = Hash::default(); + + let mut stream = StreamingPayloadStream::new( + body, + Some(SignParams { + signing_hmac, + datetime, + scope, + previous_signature: seed_signature, + }), + false, + ); + + assert!(stream.try_next().await.is_err()); + match stream.try_next().await { + Err(StreamingPayloadError::Message(msg)) if msg == "Unexpected EOF" => {} + item => panic!( + "Unexpected result, expected early EOF error, got {:?}", + item + ), + } + } +} diff --git a/src/api/generic_server.rs b/src/api/generic_server.rs deleted file mode 100644 index d0354d28..00000000 --- a/src/api/generic_server.rs +++ /dev/null @@ -1,225 +0,0 @@ -use std::net::SocketAddr; -use std::sync::Arc; - -use async_trait::async_trait; - -use futures::future::Future; - -use hyper::header::HeaderValue; -use hyper::server::conn::AddrStream; -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server}; -use hyper::{HeaderMap, StatusCode}; - -use opentelemetry::{ - global, - metrics::{Counter, ValueRecorder}, - trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, - Context, KeyValue, -}; - -use garage_util::error::Error as GarageError; -use garage_util::forwarded_headers; -use garage_util::metrics::{gen_trace_id, RecordDuration}; - -pub(crate) trait ApiEndpoint: Send + Sync + 'static { - fn name(&self) -> &'static str; - fn add_span_attributes(&self, span: SpanRef<'_>); -} - -pub trait ApiError: std::error::Error + Send + Sync + 'static { - fn http_status_code(&self) -> StatusCode; - fn add_http_headers(&self, header_map: &mut HeaderMap); - fn http_body(&self, garage_region: &str, path: &str) -> Body; -} - -#[async_trait] -pub(crate) trait ApiHandler: Send + Sync + 'static { - const API_NAME: &'static str; - const API_NAME_DISPLAY: &'static str; - - type Endpoint: ApiEndpoint; - type Error: ApiError; - - fn parse_endpoint(&self, r: &Request) -> Result; - async fn handle( - &self, - req: Request, - endpoint: Self::Endpoint, - ) -> Result, Self::Error>; -} - -pub(crate) struct ApiServer { - region: String, - api_handler: A, - - // Metrics - request_counter: Counter, - error_counter: Counter, - request_duration: ValueRecorder, -} - -impl ApiServer { - pub fn new(region: String, api_handler: A) -> Arc { - let meter = global::meter("garage/api"); - Arc::new(Self { - region, - api_handler, - request_counter: meter - .u64_counter(format!("api.{}.request_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - error_counter: meter - .u64_counter(format!("api.{}.error_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints that resulted in errors", - A::API_NAME_DISPLAY - )) - .init(), - request_duration: meter - .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) - .with_description(format!( - "Duration of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - }) - } - - pub async fn run_server( - self: Arc, - bind_addr: SocketAddr, - shutdown_signal: impl Future, - ) -> Result<(), GarageError> { - let service = make_service_fn(|conn: &AddrStream| { - let this = self.clone(); - - let client_addr = conn.remote_addr(); - async move { - Ok::<_, GarageError>(service_fn(move |req: Request| { - let this = this.clone(); - - this.handler(req, client_addr) - })) - } - }); - - let server = Server::bind(&bind_addr).serve(service); - - let graceful = server.with_graceful_shutdown(shutdown_signal); - info!( - "{} API server listening on http://{}", - A::API_NAME_DISPLAY, - bind_addr - ); - - graceful.await?; - Ok(()) - } - - async fn handler( - self: Arc, - req: Request, - addr: SocketAddr, - ) -> Result, GarageError> { - let uri = req.uri().clone(); - - if let Ok(forwarded_for_ip_addr) = - forwarded_headers::handle_forwarded_for_headers(&req.headers()) - { - info!( - "{} (via {}) {} {}", - forwarded_for_ip_addr, - addr, - req.method(), - uri - ); - } else { - info!("{} {} {}", addr, req.method(), uri); - } - debug!("{:?}", req); - - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer - .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) - .with_trace_id(gen_trace_id()) - .with_attributes(vec![ - KeyValue::new("method", format!("{}", req.method())), - KeyValue::new("uri", req.uri().to_string()), - ]) - .start(&tracer); - - let res = self - .handler_stage2(req) - .with_context(Context::current_with_span(span)) - .await; - - match res { - Ok(x) => { - debug!("{} {:?}", x.status(), x.headers()); - Ok(x) - } - Err(e) => { - let body: Body = e.http_body(&self.region, uri.path()); - let mut http_error_builder = Response::builder().status(e.http_status_code()); - - if let Some(header_map) = http_error_builder.headers_mut() { - e.add_http_headers(header_map) - } - - let http_error = http_error_builder.body(body)?; - - if e.http_status_code().is_server_error() { - warn!("Response: error {}, {}", e.http_status_code(), e); - } else { - info!("Response: error {}, {}", e.http_status_code(), e); - } - Ok(http_error) - } - } - } - - async fn handler_stage2(&self, req: Request) -> Result, A::Error> { - let endpoint = self.api_handler.parse_endpoint(&req)?; - debug!("Endpoint: {}", endpoint.name()); - - let current_context = Context::current(); - let current_span = current_context.span(); - current_span.update_name::(format!( - "{} API {}", - A::API_NAME_DISPLAY, - endpoint.name() - )); - current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); - endpoint.add_span_attributes(current_span); - - let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; - - let res = self - .api_handler - .handle(req, endpoint) - .record_duration(&self.request_duration, &metrics_tags[..]) - .await; - - self.request_counter.add(1, &metrics_tags[..]); - - let status_code = match &res { - Ok(r) => r.status(), - Err(e) => e.http_status_code(), - }; - if status_code.is_client_error() || status_code.is_server_error() { - self.error_counter.add( - 1, - &[ - metrics_tags[0].clone(), - KeyValue::new("status_code", status_code.as_str().to_string()), - ], - ); - } - - res - } -} diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml new file mode 100644 index 00000000..28f74ea3 --- /dev/null +++ b/src/api/k2v/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "garage_api_k2v" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "K2V API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model = { workspace = true, features = [ "k2v" ] } +garage_table.workspace = true +garage_util = { workspace = true, features = [ "k2v" ] } +garage_api_common.workspace = true + +base64.workspace = true +thiserror.workspace = true +tracing.workspace = true + +futures.workspace = true +tokio.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +percent-encoding.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index bb85b2e7..8e10d9a6 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -1,35 +1,34 @@ -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; - -use futures::future::Future; -use hyper::{Body, Method, Request, Response}; +use hyper::{body::Incoming as IncomingBody, Method, Request, Response}; +use tokio::sync::watch; use opentelemetry::{trace::SpanRef, KeyValue}; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; -use crate::generic_server::*; -use crate::k2v::error::*; +use garage_api_common::cors::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_request; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::batch::*; +use crate::error::*; +use crate::index::*; +use crate::item::*; +use crate::router::Endpoint; -use crate::helpers::*; -use crate::k2v::batch::*; -use crate::k2v::index::*; -use crate::k2v::item::*; -use crate::k2v::router::Endpoint; -use crate::s3::cors::*; +pub use garage_api_common::signature::streaming::ReqBody; +pub type ResBody = BoxBody; pub struct K2VApiServer { garage: Arc, } -pub(crate) struct K2VApiEndpoint { +pub struct K2VApiEndpoint { bucket_name: String, endpoint: Endpoint, } @@ -37,17 +36,16 @@ pub(crate) struct K2VApiEndpoint { impl K2VApiServer { pub async fn run( garage: Arc, - bind_addr: SocketAddr, + bind_addr: UnixOrTCPSocketAddress, s3_region: String, - shutdown_signal: impl Future, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { ApiServer::new(s3_region, K2VApiServer { garage }) - .run_server(bind_addr, shutdown_signal) + .run_server(bind_addr, None, must_exit) .await } } -#[async_trait] impl ApiHandler for K2VApiServer { const API_NAME: &'static str = "k2v"; const API_NAME_DISPLAY: &'static str = "K2V"; @@ -55,7 +53,7 @@ impl ApiHandler for K2VApiServer { type Endpoint = K2VApiEndpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { + fn parse_endpoint(&self, req: &Request) -> Result { let (endpoint, bucket_name) = Endpoint::from_request(req)?; Ok(K2VApiEndpoint { @@ -66,42 +64,38 @@ impl ApiHandler for K2VApiServer { async fn handle( &self, - req: Request, + req: Request, endpoint: K2VApiEndpoint, - ) -> Result, Error> { + ) -> Result, Error> { let K2VApiEndpoint { bucket_name, endpoint, } = endpoint; let garage = self.garage.clone(); - // The OPTIONS method is procesed early, before we even check for an API key + // The OPTIONS method is processed early, before we even check for an API key if let Endpoint::Options = endpoint { - return Ok(handle_options_s3api(garage, &req, Some(bucket_name)) + let options_res = handle_options_api(garage, &req, Some(bucket_name)) .await - .ok_or_bad_request("Error handling OPTIONS")?); + .ok_or_bad_request("Error handling OPTIONS")?; + return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "k2v", - )?; + let verified_request = verify_request(&garage, req, "k2v").await?; + let req = verified_request.request; + let api_key = verified_request.access_key; let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) - .await?; + .await + .map_err(helper_error_as_internal)?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -119,40 +113,42 @@ impl ApiHandler for K2VApiServer { // are always preflighted, i.e. the browser should make // an OPTIONS call before to check it is allowed let matching_cors_rule = match *req.method() { - Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req) - .ok_or_internal_error("Error looking up CORS rule")?, + Method::GET | Method::HEAD | Method::POST => { + find_matching_cors_rule(&bucket_params, &req) + .ok_or_internal_error("Error looking up CORS rule")? + .cloned() + } _ => None, }; + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + let resp = match endpoint { Endpoint::DeleteItem { partition_key, sort_key, - } => handle_delete_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_delete_item(ctx, req, &partition_key, &sort_key).await, Endpoint::InsertItem { partition_key, sort_key, - } => handle_insert_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_insert_item(ctx, req, &partition_key, &sort_key).await, Endpoint::ReadItem { partition_key, sort_key, - } => handle_read_item(garage, &req, bucket_id, &partition_key, &sort_key).await, + } => handle_read_item(ctx, &req, &partition_key, &sort_key).await, Endpoint::PollItem { partition_key, sort_key, causality_token, timeout, } => { - handle_poll_item( - garage, - &req, - bucket_id, - partition_key, - sort_key, - causality_token, - timeout, - ) - .await + handle_poll_item(ctx, &req, partition_key, sort_key, causality_token, timeout).await } Endpoint::ReadIndex { prefix, @@ -160,12 +156,12 @@ impl ApiHandler for K2VApiServer { end, limit, reverse, - } => handle_read_index(garage, bucket_id, prefix, start, end, limit, reverse).await, - Endpoint::InsertBatch {} => handle_insert_batch(garage, bucket_id, req).await, - Endpoint::ReadBatch {} => handle_read_batch(garage, bucket_id, req).await, - Endpoint::DeleteBatch {} => handle_delete_batch(garage, bucket_id, req).await, + } => handle_read_index(ctx, prefix, start, end, limit, reverse).await, + Endpoint::InsertBatch {} => handle_insert_batch(ctx, req).await, + Endpoint::ReadBatch {} => handle_read_batch(ctx, req).await, + Endpoint::DeleteBatch {} => handle_delete_batch(ctx, req).await, Endpoint::PollRange { partition_key } => { - handle_poll_range(garage, bucket_id, &partition_key, req).await + handle_poll_range(ctx, &partition_key, req).await } Endpoint::Options => unreachable!(), }; @@ -174,12 +170,18 @@ impl ApiHandler for K2VApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } Ok(resp_ok) } + + fn key_id_from_request(&self, req: &Request) -> Option { + garage_api_common::signature::payload::Authorization::parse_header(req.headers()) + .map(|auth| auth.key_id) + .ok() + } } impl ApiEndpoint for K2VApiEndpoint { diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 26d678da..7a03d836 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -1,31 +1,30 @@ -use std::sync::Arc; - use base64::prelude::*; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use garage_util::data::*; - use garage_table::{EnumerationOrder, TableSchema}; -use garage_model::garage::Garage; -use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::helpers::*; -use crate::k2v::error::*; -use crate::k2v::range::read_range; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::item::parse_causality_token; +use crate::range::read_range; pub async fn handle_insert_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let items = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let items = req.into_body().json::>().await?; let mut items2 = vec![]; for it in items { - let ct = it.ct.map(|s| CausalContext::parse_helper(&s)).transpose()?; + let ct = it.ct.map(|s| parse_causality_token(&s)).transpose()?; let v = match it.v { Some(vs) => DvvsValue::Value( BASE64_STANDARD @@ -37,24 +36,23 @@ pub async fn handle_insert_batch( items2.push((it.pk, it.sk, ct, v)); } - garage.k2v.rpc.insert_batch(bucket_id, items2).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items2).await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_read_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let queries = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let queries = req.into_body().json::>().await?; let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_read_batch_query(&garage, bucket_id, q)), + .map(|q| handle_read_batch_query(&ctx, q)), ) .await; @@ -67,12 +65,15 @@ pub async fn handle_read_batch( } async fn handle_read_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: ReadBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -137,16 +138,15 @@ async fn handle_read_batch_query( } pub async fn handle_delete_batch( - garage: Arc, - bucket_id: Uuid, - req: Request, -) -> Result, Error> { - let queries = parse_json_body::>(req).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let queries = req.into_body().json::>().await?; let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_delete_batch_query(&garage, bucket_id, q)), + .map(|q| handle_delete_batch_query(&ctx, q)), ) .await; @@ -159,12 +159,15 @@ pub async fn handle_delete_batch( } async fn handle_delete_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: DeleteBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -194,7 +197,7 @@ async fn handle_delete_batch_query( .k2v .rpc .insert( - bucket_id, + *bucket_id, i.partition.partition_key, i.sort_key, Some(cc), @@ -234,7 +237,7 @@ async fn handle_delete_batch_query( .collect::>(); let n = items.len(); - garage.k2v.rpc.insert_batch(bucket_id, items).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items).await?; n }; @@ -250,14 +253,16 @@ async fn handle_delete_batch_query( } pub(crate) async fn handle_poll_range( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, partition_key: &str, - req: Request, -) -> Result, Error> { + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; use garage_model::k2v::sub::PollRange; - let query = parse_json_body::(req).await?; + let query = req.into_body().json::().await?; let timeout_msec = query.timeout.unwrap_or(300).clamp(1, 600) * 1000; @@ -277,13 +282,14 @@ pub(crate) async fn handle_poll_range( query.seen_marker, timeout_msec, ) - .await?; + .await + .map_err(pass_helper_error)?; if let Some((items, seen_marker)) = resp { let resp = PollRangeResponse { items: items - .into_iter() - .map(|(_k, i)| ReadBatchResponseItem::from(i)) + .into_values() + .map(ReadBatchResponseItem::from) .collect::>(), seen_marker, }; @@ -292,7 +298,7 @@ pub(crate) async fn handle_poll_range( } else { Ok(Response::builder() .status(StatusCode::NOT_MODIFIED) - .body(Body::empty())?) + .body(empty_body())?) } } diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 4eb017ab..f1937fe5 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -1,70 +1,54 @@ -use err_derive::Error; use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; -use garage_model::helper::error::Error as HelperError; - -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::CustomApiErrorBody; -use crate::signature::error::Error as SignatureError; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; +pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error}; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), + /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// Some base64 encoded data was badly encoded - #[error(display = "Invalid base64: {}", _0)] - InvalidBase64(#[error(source)] base64::DecodeError), + #[error("Invalid base64: {0}")] + InvalidBase64(#[from] base64::DecodeError), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// Invalid causality token + #[error("Invalid causality token")] + InvalidCausalityToken, /// The client asked for an invalid return format (invalid Accept header) - #[error(display = "Not acceptable: {}", _0)] + #[error("Not acceptable: {0}")] NotAcceptable(String), /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} - -impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - e => Self::Common(CommonError::BadRequest(format!("{}", e))), - } - } -} +commonErrorDerivative!(Error); impl From for Error { fn from(err: SignatureError) -> Self { @@ -74,7 +58,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), - SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } @@ -90,8 +74,9 @@ impl Error { Error::NotAcceptable(_) => "NotAcceptable", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", Error::InvalidBase64(_) => "InvalidBase64", - Error::InvalidHeader(_) => "InvalidHeaderValue", Error::InvalidUtf8Str(_) => "InvalidUtf8String", + Error::InvalidCausalityToken => "CausalityToken", + Error::InvalidDigest(_) => "InvalidDigest", } } } @@ -105,8 +90,9 @@ impl ApiError for Error { Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, Error::AuthorizationHeaderMalformed(_) | Error::InvalidBase64(_) - | Error::InvalidHeader(_) - | Error::InvalidUtf8Str(_) => StatusCode::BAD_REQUEST, + | Error::InvalidUtf8Str(_) + | Error::InvalidDigest(_) + | Error::InvalidCausalityToken => StatusCode::BAD_REQUEST, } } @@ -115,14 +101,14 @@ impl ApiError for Error { header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap()); } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = CustomApiErrorBody { code: self.code().to_string(), message: format!("{}", self), path: path.to_string(), region: garage_region.to_string(), }; - Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + let error_str = serde_json::to_string_pretty(&error).unwrap_or_else(|_| { r#" { "code": "InternalError", @@ -130,6 +116,7 @@ impl ApiError for Error { } "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index 6c1d4a91..fbfaad98 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -1,32 +1,35 @@ -use std::sync::Arc; - -use hyper::{Body, Response}; +use hyper::Response; use serde::Serialize; -use garage_util::data::*; - -use garage_rpc::ring::Ring; use garage_table::util::*; -use garage_model::garage::Garage; use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; -use crate::helpers::*; -use crate::k2v::error::*; -use crate::k2v::range::read_range; +use garage_api_common::helpers::*; + +use crate::api_server::ResBody; +use crate::error::*; +use crate::range::read_range; pub async fn handle_read_index( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, prefix: Option, start: Option, end: Option, limit: Option, reverse: Option, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let reverse = reverse.unwrap_or(false); - let ring: Arc = garage.system.ring.borrow().clone(); + let node_id_vec = garage + .system + .cluster_layout() + .all_nongateway_nodes() + .to_vec(); let (partition_keys, more, next_start) = read_range( &garage.k2v.counter_table.table, @@ -35,7 +38,7 @@ pub async fn handle_read_index( &start, &end, limit, - Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())), + Some((DeletedFilter::NotDeleted, node_id_vec)), EnumerationOrder::from_reverse(reverse), ) .await?; @@ -54,7 +57,7 @@ pub async fn handle_read_index( partition_keys: partition_keys .into_iter() .map(|part| { - let vals = part.filtered_values(&ring); + let vals = part.filtered_values(&garage.system.cluster_layout()); ReadIndexResponseEntry { pk: part.sk, entries: *vals.get(&s_entries).unwrap_or(&0), @@ -68,7 +71,7 @@ pub async fn handle_read_index( next_start, }; - Ok(json_ok_response(&resp)?) + json_ok_response::(&resp) } #[derive(Serialize)] diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index e13a0f30..0fb945d2 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -1,17 +1,15 @@ -use std::sync::Arc; - use base64::prelude::*; use http::header; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; -use garage_util::data::*; - -use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::k2v::error::*; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; @@ -21,8 +19,12 @@ pub enum ReturnFormat { Either, } +pub(crate) fn parse_causality_token(s: &str) -> Result { + CausalContext::parse(s).ok_or(Error::InvalidCausalityToken) +} + impl ReturnFormat { - pub fn from(req: &Request) -> Result { + pub fn from(req: &Request) -> Result { let accept = match req.headers().get(header::ACCEPT) { Some(a) => a.to_str()?, None => return Ok(Self::Json), @@ -40,7 +42,7 @@ impl ReturnFormat { } } - pub fn make_response(&self, item: &K2VItem) -> Result, Error> { + pub fn make_response(&self, item: &K2VItem) -> Result, Error> { let vals = item.values(); if vals.is_empty() { @@ -52,7 +54,7 @@ impl ReturnFormat { Self::Binary if vals.len() > 1 => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .status(StatusCode::CONFLICT) - .body(Body::empty())?), + .body(empty_body())?), Self::Binary => { assert!(vals.len() == 1); Self::make_binary_response(ct, vals[0]) @@ -62,22 +64,22 @@ impl ReturnFormat { } } - fn make_binary_response(ct: String, v: &DvvsValue) -> Result, Error> { + fn make_binary_response(ct: String, v: &DvvsValue) -> Result, Error> { match v { DvvsValue::Deleted => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/octet-stream") .status(StatusCode::NO_CONTENT) - .body(Body::empty())?), + .body(empty_body())?), DvvsValue::Value(v) => Ok(Response::builder() .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/octet-stream") .status(StatusCode::OK) - .body(Body::from(v.to_vec()))?), + .body(bytes_body(v.to_vec().into()))?), } } - fn make_json_response(ct: String, v: &[&DvvsValue]) -> Result, Error> { + fn make_json_response(ct: String, v: &[&DvvsValue]) -> Result, Error> { let items = v .iter() .map(|v| match v { @@ -91,19 +93,22 @@ impl ReturnFormat { .header(X_GARAGE_CAUSALITY_TOKEN, ct) .header(header::CONTENT_TYPE, "application/json") .status(StatusCode::OK) - .body(Body::from(json_body))?) + .body(string_body(json_body))?) } } /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_read_item( - garage: Arc, - req: &Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, partition_key: &str, sort_key: &String, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let format = ReturnFormat::from(req)?; let item = garage @@ -111,7 +116,7 @@ pub async fn handle_read_item( .item_table .get( &K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: partition_key.to_string(), }, sort_key, @@ -123,28 +128,31 @@ pub async fn handle_read_item( } pub async fn handle_insert_item( - garage: Arc, - req: Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: Request, partition_key: &str, sort_key: &str, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) .map(|s| s.to_str()) .transpose()? - .map(CausalContext::parse_helper) + .map(parse_causality_token) .transpose()?; - let body = hyper::body::to_bytes(req.into_body()).await?; + let body = req.into_body().collect().await?; + let value = DvvsValue::Value(body.to_vec()); garage .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -154,22 +162,24 @@ pub async fn handle_insert_item( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_delete_item( - garage: Arc, - req: Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: Request, partition_key: &str, sort_key: &str, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) .map(|s| s.to_str()) .transpose()? - .map(CausalContext::parse_helper) + .map(parse_causality_token) .transpose()?; let value = DvvsValue::Deleted; @@ -178,7 +188,7 @@ pub async fn handle_delete_item( .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -188,20 +198,22 @@ pub async fn handle_delete_item( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_poll_item( - garage: Arc, - req: &Request, - bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, partition_key: String, sort_key: String, causality_token: String, timeout_secs: Option, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let format = ReturnFormat::from(req)?; let causal_context = @@ -213,7 +225,7 @@ pub async fn handle_poll_item( .k2v .rpc .poll_item( - bucket_id, + *bucket_id, partition_key, sort_key, causal_context, @@ -226,6 +238,6 @@ pub async fn handle_poll_item( } else { Ok(Response::builder() .status(StatusCode::NOT_MODIFIED) - .body(Body::empty())?) + .body(empty_body())?) } } diff --git a/src/api/k2v/mod.rs b/src/api/k2v/lib.rs similarity index 70% rename from src/api/k2v/mod.rs rename to src/api/k2v/lib.rs index b6a8c5cf..334ae46b 100644 --- a/src/api/k2v/mod.rs +++ b/src/api/k2v/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; mod error; mod router; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index bb9d3be5..eb4738db 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::helpers::key_after_prefix; -use crate::k2v::error::*; +use garage_api_common::helpers::key_after_prefix; + +use crate::error::*; /// Read range in a Garage table. /// Returns (entries, more?, nextStart) diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index 1cc58be5..a04b0f81 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -1,11 +1,11 @@ -use crate::k2v::error::*; +use crate::error::*; use std::borrow::Cow; use hyper::{Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; router_match! {@func diff --git a/src/api/lib.rs b/src/api/lib.rs deleted file mode 100644 index 370dfd7a..00000000 --- a/src/api/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Crate for serving a S3 compatible API -#[macro_use] -extern crate tracing; - -pub mod common_error; - -mod encoding; -pub mod generic_server; -pub mod helpers; -mod router_macros; -/// This mode is public only to help testing. Don't expect stability here -pub mod signature; - -pub mod admin; -#[cfg(feature = "k2v")] -pub mod k2v; -pub mod s3; diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml new file mode 100644 index 00000000..88630866 --- /dev/null +++ b/src/api/s3/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "garage_api_s3" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +aes-gcm.workspace = true +async-compression.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +thiserror.workspace = true +hex.workspace = true +tracing.workspace = true +md-5.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 27837297..acb0cf56 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -1,42 +1,44 @@ -use std::net::SocketAddr; use std::sync::Arc; -use async_trait::async_trait; - -use futures::future::Future; use hyper::header; -use hyper::{Body, Request, Response}; +use hyper::{body::Incoming as IncomingBody, Request, Response}; +use tokio::sync::watch; use opentelemetry::{trace::SpanRef, KeyValue}; use garage_util::error::Error as GarageError; +use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::generic_server::*; -use crate::s3::error::*; +use garage_api_common::cors::*; +use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_request; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::bucket::*; +use crate::copy::*; +use crate::cors::*; +use crate::delete::*; +use crate::error::*; +use crate::get::*; +use crate::lifecycle::*; +use crate::list::*; +use crate::multipart::*; +use crate::post_object::handle_post_object; +use crate::put::*; +use crate::router::Endpoint; +use crate::website::*; -use crate::helpers::*; -use crate::s3::bucket::*; -use crate::s3::copy::*; -use crate::s3::cors::*; -use crate::s3::delete::*; -use crate::s3::get::*; -use crate::s3::list::*; -use crate::s3::post_object::handle_post_object; -use crate::s3::put::*; -use crate::s3::router::Endpoint; -use crate::s3::website::*; +pub use garage_api_common::signature::streaming::ReqBody; +pub type ResBody = BoxBody; pub struct S3ApiServer { garage: Arc, } -pub(crate) struct S3ApiEndpoint { +pub struct S3ApiEndpoint { bucket_name: Option, endpoint: Endpoint, } @@ -44,21 +46,21 @@ pub(crate) struct S3ApiEndpoint { impl S3ApiServer { pub async fn run( garage: Arc, - addr: SocketAddr, + addr: UnixOrTCPSocketAddress, s3_region: String, - shutdown_signal: impl Future, + must_exit: watch::Receiver, ) -> Result<(), GarageError> { ApiServer::new(s3_region, S3ApiServer { garage }) - .run_server(addr, shutdown_signal) + .run_server(addr, None, must_exit) .await } async fn handle_request_without_bucket( &self, - _req: Request, + _req: Request, api_key: Key, endpoint: Endpoint, - ) -> Result, Error> { + ) -> Result, Error> { match endpoint { Endpoint::ListBuckets => handle_list_buckets(&self.garage, &api_key).await, endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), @@ -66,7 +68,6 @@ impl S3ApiServer { } } -#[async_trait] impl ApiHandler for S3ApiServer { const API_NAME: &'static str = "s3"; const API_NAME_DISPLAY: &'static str = "S3"; @@ -74,7 +75,7 @@ impl ApiHandler for S3ApiServer { type Endpoint = S3ApiEndpoint; type Error = Error; - fn parse_endpoint(&self, req: &Request) -> Result { + fn parse_endpoint(&self, req: &Request) -> Result { let authority = req .headers() .get(header::HOST) @@ -102,9 +103,9 @@ impl ApiHandler for S3ApiServer { async fn handle( &self, - req: Request, + req: Request, endpoint: S3ApiEndpoint, - ) -> Result, Error> { + ) -> Result, Error> { let S3ApiEndpoint { bucket_name, endpoint, @@ -116,20 +117,13 @@ impl ApiHandler for S3ApiServer { return handle_post_object(garage, req, bucket_name.unwrap()).await; } if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, bucket_name).await; + let options_res = handle_options_api(garage, &req, bucket_name).await?; + return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "s3", - )?; + let verified_request = verify_request(&garage, req, "s3").await?; + let req = verified_request.request; + let api_key = verified_request.access_key; let bucket_name = match bucket_name { None => { @@ -142,17 +136,19 @@ impl ApiHandler for S3ApiServer { // Special code path for CreateBucket API endpoint if let Endpoint::CreateBucket {} = endpoint { - return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await; + return handle_create_bucket(&garage, req, &api_key.key_id, bucket_name).await; } let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -165,83 +161,72 @@ impl ApiHandler for S3ApiServer { return Err(Error::forbidden("Operation is not allowed for this key.")); } - let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?; + let matching_cors_rule = find_matching_cors_rule(&bucket_params, &req)?.cloned(); + + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; let resp = match endpoint { Endpoint::HeadObject { key, part_number, .. - } => handle_head(garage, &req, bucket_id, &key, part_number).await, + } => handle_head(ctx, &req.map(|_| ()), &key, part_number).await, Endpoint::GetObject { - key, part_number, .. - } => handle_get(garage, &req, bucket_id, &key, part_number).await, + key, + part_number, + response_cache_control, + response_content_disposition, + response_content_encoding, + response_content_language, + response_content_type, + response_expires, + .. + } => { + let overrides = GetObjectOverrides { + response_cache_control, + response_content_disposition, + response_content_encoding, + response_content_language, + response_content_type, + response_expires, + }; + handle_get(ctx, &req.map(|_| ()), &key, part_number, overrides).await + } Endpoint::UploadPart { key, part_number, upload_id, - } => { - handle_put_part( - garage, - req, - bucket_id, - &key, - part_number, - &upload_id, - content_sha256, - ) - .await - } - Endpoint::CopyObject { key } => { - handle_copy(garage, &api_key, &req, bucket_id, &key).await - } + } => handle_put_part(ctx, req, &key, part_number, &upload_id).await, + Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await, Endpoint::UploadPartCopy { key, part_number, upload_id, - } => { - handle_upload_part_copy( - garage, - &api_key, - &req, - bucket_id, - &key, - part_number, - &upload_id, - ) - .await - } - Endpoint::PutObject { key } => { - handle_put(garage, req, &bucket, &key, content_sha256).await - } + } => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await, + Endpoint::PutObject { key } => handle_put(ctx, req, &key).await, Endpoint::AbortMultipartUpload { key, upload_id } => { - handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await + handle_abort_multipart_upload(ctx, &key, &upload_id).await } - Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await, + Endpoint::DeleteObject { key, .. } => handle_delete(ctx, &key).await, Endpoint::CreateMultipartUpload { key } => { - handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await + handle_create_multipart_upload(ctx, &req, &key).await } Endpoint::CompleteMultipartUpload { key, upload_id } => { - handle_complete_multipart_upload( - garage, - req, - &bucket_name, - &bucket, - &key, - &upload_id, - content_sha256, - ) - .await + handle_complete_multipart_upload(ctx, req, &key, &upload_id).await } Endpoint::CreateBucket {} => unreachable!(), Endpoint::HeadBucket {} => { - let empty_body: Body = Body::from(vec![]); - let response = Response::builder().body(empty_body).unwrap(); + let response = Response::builder().body(empty_body()).unwrap(); Ok(response) } - Endpoint::DeleteBucket {} => { - handle_delete_bucket(&garage, bucket_id, bucket_name, api_key).await - } - Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage), + Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await, + Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx), Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(), + Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx), Endpoint::ListObjects { delimiter, encoding_type, @@ -249,24 +234,21 @@ impl ApiHandler for S3ApiServer { max_keys, prefix, } => { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_keys.map(|p| p.clamp(1, 1000)).unwrap_or(1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - is_v2: false, - marker, - continuation_token: None, - start_after: None, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + is_v2: false, + marker, + continuation_token: None, + start_after: None, + }; + handle_list(ctx, &query).await } Endpoint::ListObjectsV2 { delimiter, @@ -279,24 +261,21 @@ impl ApiHandler for S3ApiServer { .. } => { if list_type == "2" { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_keys.map(|p| p.clamp(1, 1000)).unwrap_or(1000), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - prefix: prefix.unwrap_or_default(), - }, - is_v2: true, - marker: None, - continuation_token, - start_after, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), + prefix: prefix.unwrap_or_default(), }, - ) - .await + is_v2: true, + marker: None, + continuation_token, + start_after, + }; + handle_list(ctx, &query).await } else { Err(Error::bad_request(format!( "Invalid endpoint: list-type={}", @@ -312,22 +291,19 @@ impl ApiHandler for S3ApiServer { prefix, upload_id_marker, } => { - handle_list_multipart_upload( - garage, - &ListMultipartUploadsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter: delimiter.map(|d| d.to_string()), - page_size: max_uploads.map(|p| p.clamp(1, 1000)).unwrap_or(1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - key_marker, - upload_id_marker, + let query = ListMultipartUploadsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_uploads.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + key_marker, + upload_id_marker, + }; + handle_list_multipart_upload(ctx, &query).await } Endpoint::ListParts { key, @@ -335,32 +311,25 @@ impl ApiHandler for S3ApiServer { part_number_marker, upload_id, } => { - handle_list_parts( - garage, - &ListPartsQuery { - bucket_name, - bucket_id, - key, - upload_id, - part_number_marker: part_number_marker.map(|p| p.clamp(1, 10000)), - max_parts: max_parts.map(|p| p.clamp(1, 1000)).unwrap_or(1000), - }, - ) - .await + let query = ListPartsQuery { + bucket_name: ctx.bucket_name.clone(), + key, + upload_id, + part_number_marker: part_number_marker.map(|p| p.min(10000)), + max_parts: max_parts.unwrap_or(1000).clamp(1, 1000), + }; + handle_list_parts(ctx, req, &query).await } - Endpoint::DeleteObjects {} => { - handle_delete_objects(garage, bucket_id, req, content_sha256).await - } - Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await, - Endpoint::PutBucketWebsite {} => { - handle_put_website(garage, bucket_id, req, content_sha256).await - } - Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket_id).await, - Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await, - Endpoint::PutBucketCors {} => { - handle_put_cors(garage, bucket_id, req, content_sha256).await - } - Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket_id).await, + Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req).await, + Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await, + Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req).await, + Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await, + Endpoint::GetBucketCors {} => handle_get_cors(ctx).await, + Endpoint::PutBucketCors {} => handle_put_cors(ctx, req).await, + Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await, + Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await, + Endpoint::PutBucketLifecycleConfiguration {} => handle_put_lifecycle(ctx, req).await, + Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await, endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), }; @@ -368,12 +337,18 @@ impl ApiHandler for S3ApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } Ok(resp_ok) } + + fn key_id_from_request(&self, req: &Request) -> Option { + garage_api_common::signature::payload::Authorization::parse_header(req.headers()) + .map(|auth| auth.key_id) + .ok() + } } impl ApiEndpoint for S3ApiEndpoint { diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 733981e1..55caa6c8 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -1,24 +1,25 @@ use std::collections::HashMap; -use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use garage_model::bucket_alias_table::*; use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; -use garage_model::key_table::Key; +use garage_model::key_table::{Key, KeyParams}; use garage_model::permission::BucketKeyPerm; use garage_table::util::*; use garage_util::crdt::*; -use garage_util::data::*; use garage_util::time::*; -use crate::common_error::CommonError; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; -pub fn handle_get_bucket_location(garage: Arc) -> Result, Error> { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; + +pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { garage, .. } = ctx; let loc = s3_xml::LocationConstraint { xmlns: (), region: garage.config.s3_api.s3_region.to_string(), @@ -27,10 +28,10 @@ pub fn handle_get_bucket_location(garage: Arc) -> Result, Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } -pub fn handle_get_bucket_versioning() -> Result, Error> { +pub fn handle_get_bucket_versioning() -> Result, Error> { let versioning = s3_xml::VersioningConfiguration { xmlns: (), status: None, @@ -40,10 +41,62 @@ pub fn handle_get_bucket_versioning() -> Result, Error> { Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } -pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result, Error> { +pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + bucket_id, api_key, .. + } = ctx; + let key_p = api_key.params().ok_or_internal_error( + "Key should not be in deleted state at this point (in handle_get_bucket_acl)", + )?; + + let mut grants: Vec = vec![]; + let kp = api_key.bucket_permissions(&bucket_id); + + if kp.allow_owner { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("FULL_CONTROL".to_string()), + }); + } else { + if kp.allow_read { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ".to_string()), + }); + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ_ACP".to_string()), + }); + } + if kp.allow_write { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("WRITE".to_string()), + }); + } + } + + let access_control_policy = s3_xml::AccessControlPolicy { + xmlns: (), + owner: None, + acl: s3_xml::AccessControlList { entries: grants }, + }; + + let xml = s3_xml::to_xml_with_header(&access_control_policy)?; + trace!("xml: {}", xml); + + Ok(Response::builder() + .header("Content-Type", "application/xml") + .body(string_body(xml))?) +} + +pub async fn handle_list_buckets( + garage: &Garage, + api_key: &Key, +) -> Result, Error> { let key_p = api_key.params().ok_or_internal_error( "Key should not be in deleted state at this point (in handle_list_buckets)", )?; @@ -109,21 +162,16 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result, - content_sha256: Option, - api_key: Key, + req: Request, + api_key_id: &String, bucket_name: String, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } +) -> Result, Error> { + let body = req.into_body().collect().await?; let cmd = parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?; @@ -138,16 +186,18 @@ pub async fn handle_create_bucket( } } - let key_params = api_key - .params() - .ok_or_internal_error("Key should not be deleted at this point")?; + let helper = garage.locked_helper().await; + + // refetch API key after taking lock to ensure up-to-date data + let api_key = helper.key().get_existing_key(api_key_id).await?; + let key_params = api_key.params().unwrap(); let existing_bucket = if let Some(Some(bucket_id)) = key_params.local_aliases.get(&bucket_name) { Some(*bucket_id) } else { - garage - .bucket_helper() + helper + .bucket() .resolve_global_bucket_name(&bucket_name) .await? }; @@ -171,7 +221,7 @@ pub async fn handle_create_bucket( } // Create the bucket! - if !is_valid_bucket_name(&bucket_name) { + if !is_valid_bucket_name(&bucket_name, garage.config.allow_punycode) { return Err(Error::bad_request(format!( "{}: {}", bucket_name, INVALID_BUCKET_NAME_MESSAGE @@ -181,40 +231,35 @@ pub async fn handle_create_bucket( let bucket = Bucket::new(); garage.bucket_table.insert(&bucket).await?; - garage - .bucket_helper() + helper .set_bucket_key_permissions(bucket.id, &api_key.key_id, BucketKeyPerm::ALL_PERMISSIONS) .await?; - garage - .bucket_helper() + helper .set_local_bucket_alias(bucket.id, &api_key.key_id, &bucket_name) .await?; } Ok(Response::builder() .header("Location", format!("/{}", bucket_name)) - .body(Body::empty()) + .body(empty_body()) .unwrap()) } -pub async fn handle_delete_bucket( - garage: &Garage, - bucket_id: Uuid, - bucket_name: String, - api_key: Key, -) -> Result, Error> { - let key_params = api_key - .params() - .ok_or_internal_error("Key should not be deleted at this point")?; +pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params: bucket_state, + api_key, + .. + } = &ctx; + let helper = garage.locked_helper().await; - let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_))); + let key_params = api_key.params().unwrap(); - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - let bucket_state = bucket.state.as_option().unwrap(); + let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_))); // If the bucket has no other aliases, this is a true deletion. // Otherwise, it is just an alias removal. @@ -224,65 +269,63 @@ pub async fn handle_delete_bucket( .items() .iter() .filter(|(_, _, active)| *active) - .any(|(n, _, _)| is_local_alias || (*n != bucket_name)); + .any(|(n, _, _)| is_local_alias || (*n != *bucket_name)); let has_other_local_aliases = bucket_state .local_aliases .items() .iter() .filter(|(_, _, active)| *active) - .any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id); + .any(|((k, n), _, _)| !is_local_alias || *n != *bucket_name || *k != api_key.key_id); if !has_other_global_aliases && !has_other_local_aliases { // Delete bucket // Check bucket is empty - if !garage.bucket_helper().is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(*bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } // --- done checking, now commit --- // 1. delete bucket alias if is_local_alias { - garage - .bucket_helper() - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + helper + .purge_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &bucket_name) + helper + .purge_global_bucket_alias(*bucket_id, bucket_name) .await?; } // 2. delete authorization from keys that had access - for (key_id, _) in bucket.authorized_keys() { - garage - .bucket_helper() - .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) + for (key_id, _) in bucket_state.authorized_keys.items() { + helper + .set_bucket_key_permissions(*bucket_id, key_id, BucketKeyPerm::NO_PERMISSIONS) .await?; } + let bucket = Bucket { + id: *bucket_id, + state: Deletable::delete(), + }; // 3. delete bucket - bucket.state = Deletable::delete(); garage.bucket_table.insert(&bucket).await?; } else if is_local_alias { // Just unalias - garage - .bucket_helper() - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + helper + .unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { // Just unalias (but from global namespace) - garage - .bucket_helper() - .unset_global_bucket_alias(bucket_id, &bucket_name) + helper + .unset_global_bucket_alias(*bucket_id, bucket_name) .await?; } Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option> { @@ -317,6 +360,15 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option> { Some(ret) } +fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee { + s3_xml::Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(s3_xml::Value(key_params.name.get().to_string())), + id: Some(s3_xml::Value(api_key.key_id.to_string())), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 7eb6459d..47a63c82 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -1,68 +1,220 @@ use std::pin::Pin; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use futures::{stream, stream::Stream, StreamExt, TryFutureExt}; -use md5::{Digest as Md5Digest, Md5}; +use futures::{stream, stream::Stream, StreamExt, TryStreamExt}; use bytes::Bytes; -use hyper::{Body, Request, Response}; +use http::header::HeaderName; +use hyper::{Request, Response}; use serde::Serialize; -use garage_rpc::netapp::bytes_buf::BytesBuf; +use garage_net::bytes_buf::BytesBuf; +use garage_net::stream::read_stream_to_end; use garage_rpc::rpc_helper::OrderTag; use garage_table::*; use garage_util::data::*; +use garage_util::error::Error as GarageError; use garage_util::time::*; -use garage_model::garage::Garage; -use garage_model::key_table::Key; use garage_model::s3::block_ref_table::*; +use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::parse_bucket_key; -use crate::s3::error::*; -use crate::s3::put::{decode_upload_id, get_headers}; -use crate::s3::xml::{self as s3_xml, xmlns_tag}; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders}; +use crate::multipart; +use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult}; +use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; +use crate::xml::{self as s3_xml, xmlns_tag}; + +pub const X_AMZ_COPY_SOURCE_IF_MATCH: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-match"); +pub const X_AMZ_COPY_SOURCE_IF_NONE_MATCH: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-none-match"); +pub const X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-modified-since"); +pub const X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE: HeaderName = + HeaderName::from_static("x-amz-copy-source-if-unmodified-since"); + +// -------- CopyObject --------- pub async fn handle_copy( - garage: Arc, - api_key: &Key, - req: &Request, - dest_bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, dest_key: &str, -) -> Result, Error> { - let copy_precondition = CopyPreconditionHeaders::parse(req)?; +) -> Result, Error> { + let copy_precondition = PreconditionHeaders::parse_copy_source(req)?; - let source_object = get_copy_source(&garage, api_key, req).await?; + let checksum_algorithm = request_checksum_algorithm(req.headers())?; + + let source_object = get_copy_source(&ctx, req).await?; let (source_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; // Check precondition, e.g. x-amz-copy-source-if-match - copy_precondition.check(source_version, &source_version_meta.etag)?; + copy_precondition.check_copy_source(source_version, &source_version_meta.etag)?; + + // Determine encryption parameters + let (source_encryption, source_object_meta_inner) = + EncryptionParams::check_decrypt_for_copy_source( + &ctx.garage, + req.headers(), + &source_version_meta.encryption, + )?; + let dest_encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?; + + // Extract source checksum info before source_object_meta_inner is consumed + let source_checksum = source_object_meta_inner.checksum; + let source_checksum_algorithm = source_checksum.map(|x| x.algorithm()); + + // If source object has a checksum, the destination object must as well. + // The x-amz-checksum-algorithm header allows to change that algorithm, + // but if it is absent, we must use the same as before + let checksum_algorithm = checksum_algorithm.or(source_checksum_algorithm); + + // Determine metadata of destination object + let was_multipart = source_version_meta.etag.contains('-'); + let dest_object_meta = ObjectVersionMetaInner { + headers: match req.headers().get("x-amz-metadata-directive") { + Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => { + extract_metadata_headers(req.headers())? + } + _ => { + // The x-amz-website-redirect-location header is not copied, instead + // it is replaced by the value from the request (or removed if no + // value was specified) + let is_redirect = + |(key, _): &(String, String)| key == X_AMZ_WEBSITE_REDIRECT_LOCATION.as_str(); + let mut headers: Vec<_> = source_object_meta_inner.headers.clone(); + headers.retain(|h| !is_redirect(h)); + let new_headers = extract_metadata_headers(req.headers())?; + headers.extend(new_headers.into_iter().filter(is_redirect)); + headers + } + }, + checksum: source_checksum, + }; + + // Do actual object copying + // + // In any of the following scenarios, we need to read the whole object + // data and re-write it again: + // + // - the data needs to be decrypted or encrypted + // - the requested checksum algorithm requires us to recompute a checksum + // - the original object was a multipart upload and a checksum algorithm + // is defined (AWS specifies that in this case, we must recompute the + // checksum from scratch as if this was a single big object and not + // a multipart object, as the checksums are not computed in the same way) + // + // In other cases, we can just copy the metadata and reference the same blocks. + // + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + + let must_recopy = !EncryptionParams::is_same(&source_encryption, &dest_encryption) + || source_checksum_algorithm != checksum_algorithm + || (was_multipart && checksum_algorithm.is_some()); + + let res = if !must_recopy { + // In most cases, we can just copy the metadata and link blocks of the + // old object from the new object. + handle_copy_metaonly( + ctx, + dest_key, + dest_object_meta, + dest_encryption, + source_version, + source_version_data, + source_version_meta, + ) + .await? + } else { + let expected_checksum = ExpectedChecksums { + md5: None, + sha256: None, + extra: source_checksum, + }; + let checksum_mode = if was_multipart || source_checksum_algorithm != checksum_algorithm { + ChecksumMode::Calculate(checksum_algorithm) + } else { + ChecksumMode::Verify(&expected_checksum) + }; + // If source and dest encryption use different keys, + // we must decrypt content and re-encrypt, so rewrite all data blocks. + handle_copy_reencrypt( + ctx, + dest_key, + dest_object_meta, + dest_encryption, + source_version, + source_version_data, + source_encryption, + checksum_mode, + ) + .await? + }; + + let last_modified = msec_to_rfc3339(res.version_timestamp); + let result = CopyObjectResult { + last_modified: s3_xml::Value(last_modified), + etag: s3_xml::Value(format!("\"{}\"", res.etag)), + }; + let xml = s3_xml::to_xml_with_header(&result)?; + + let mut resp = Response::builder() + .header("Content-Type", "application/xml") + .header("x-amz-version-id", hex::encode(res.version_uuid)) + .header( + "x-amz-copy-source-version-id", + hex::encode(source_version.uuid), + ); + dest_encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(xml))?) +} + +async fn handle_copy_metaonly( + ctx: ReqCtx, + dest_key: &str, + dest_object_meta: ObjectVersionMetaInner, + dest_encryption: EncryptionParams, + source_version: &ObjectVersion, + source_version_data: &ObjectVersionData, + source_version_meta: &ObjectVersionMeta, +) -> Result { + let ReqCtx { + garage, + bucket_id: dest_bucket_id, + .. + } = ctx; // Generate parameters for copied object let new_uuid = gen_uuid(); let new_timestamp = now_msec(); - // Implement x-amz-metadata-directive: REPLACE - let new_meta = match req.headers().get("x-amz-metadata-directive") { - Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => ObjectVersionMeta { - headers: get_headers(req.headers())?, - size: source_version_meta.size, - etag: source_version_meta.etag.clone(), - }, - _ => source_version_meta.clone(), + let new_meta = ObjectVersionMeta { + encryption: dest_encryption.encrypt_meta(dest_object_meta)?, + size: source_version_meta.size, + etag: source_version_meta.etag.clone(), }; - let etag = new_meta.etag.to_string(); + let res = SaveStreamResult { + version_uuid: new_uuid, + version_timestamp: new_timestamp, + etag: new_meta.etag.clone(), + }; // Save object copy match source_version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_meta, bytes) => { + // bytes is either plaintext before&after or encrypted with the + // same keys, so it's ok to just copy it as is let dest_object_version = ObjectVersion { uuid: new_uuid, timestamp: new_timestamp, @@ -85,6 +237,7 @@ pub async fn handle_copy( .get(&source_version.uuid, &EmptyKey) .await?; let source_version = source_version.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; // Write an "uploading" marker in Object table // This holds a reference to the object in the Version table @@ -92,7 +245,11 @@ pub async fn handle_copy( let tmp_dest_object_version = ObjectVersion { uuid: new_uuid, timestamp: new_timestamp, - state: ObjectVersionState::Uploading(new_meta.headers.clone()), + state: ObjectVersionState::Uploading { + encryption: new_meta.encryption.clone(), + checksum_algorithm: None, + multipart: false, + }, }; let tmp_dest_object = Object::new( dest_bucket_id, @@ -105,8 +262,14 @@ pub async fn handle_copy( // this means that the BlockRef entries linked to this version cannot be // marked as deleted (they are marked as deleted only if the Version // doesn't exist or is marked as deleted). - let mut dest_version = - Version::new(new_uuid, dest_bucket_id, dest_key.to_string(), false); + let mut dest_version = Version::new( + new_uuid, + VersionBacklink::Object { + bucket_id: dest_bucket_id, + key: dest_key.to_string(), + }, + false, + ); garage.version_table.insert(&dest_version).await?; // Fill in block list for version and insert block refs @@ -151,51 +314,84 @@ pub async fn handle_copy( } } - let last_modified = msec_to_rfc3339(new_timestamp); - let result = CopyObjectResult { - last_modified: s3_xml::Value(last_modified), - etag: s3_xml::Value(format!("\"{}\"", etag)), - }; - let xml = s3_xml::to_xml_with_header(&result)?; - - Ok(Response::builder() - .header("Content-Type", "application/xml") - .header("x-amz-version-id", hex::encode(new_uuid)) - .header( - "x-amz-copy-source-version-id", - hex::encode(source_version.uuid), - ) - .body(Body::from(xml))?) + Ok(res) } +async fn handle_copy_reencrypt( + ctx: ReqCtx, + dest_key: &str, + dest_object_meta: ObjectVersionMetaInner, + dest_encryption: EncryptionParams, + source_version: &ObjectVersion, + source_version_data: &ObjectVersionData, + source_encryption: EncryptionParams, + checksum_mode: ChecksumMode<'_>, +) -> Result { + // basically we will read the source data (decrypt if necessary) + // and save that in a new object (encrypt if necessary), + // by combining the code used in getobject and putobject + let source_stream = full_object_byte_stream( + ctx.garage.clone(), + source_version, + source_version_data, + source_encryption, + ); + + save_stream( + &ctx, + dest_object_meta, + dest_encryption, + source_stream.map_err(|e| Error::from(GarageError::from(e))), + &dest_key.to_string(), + checksum_mode, + ) + .await +} + +// -------- UploadPartCopy --------- + pub async fn handle_upload_part_copy( - garage: Arc, - api_key: &Key, - req: &Request, - dest_bucket_id: Uuid, + ctx: ReqCtx, + req: &Request, dest_key: &str, part_number: u64, upload_id: &str, -) -> Result, Error> { - let copy_precondition = CopyPreconditionHeaders::parse(req)?; +) -> Result, Error> { + let copy_precondition = PreconditionHeaders::parse_copy_source(req)?; - let dest_version_uuid = decode_upload_id(upload_id)?; + let dest_upload_id = multipart::decode_upload_id(upload_id)?; let dest_key = dest_key.to_string(); - let (source_object, dest_object) = futures::try_join!( - get_copy_source(&garage, api_key, req), - garage - .object_table - .get(&dest_bucket_id, &dest_key) - .map_err(Error::from), + let (source_object, (_, dest_version, mut dest_mpu)) = futures::try_join!( + get_copy_source(&ctx, req), + multipart::get_upload(&ctx, &dest_key, &dest_upload_id) )?; - let dest_object = dest_object.ok_or(Error::NoSuchKey)?; + + let ReqCtx { garage, .. } = ctx; let (source_object_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; // Check precondition on source, e.g. x-amz-copy-source-if-match - copy_precondition.check(source_object_version, &source_version_meta.etag)?; + copy_precondition.check_copy_source(source_object_version, &source_version_meta.etag)?; + + // Determine encryption parameters + let (source_encryption, _) = EncryptionParams::check_decrypt_for_copy_source( + &garage, + req.headers(), + &source_version_meta.encryption, + )?; + let (dest_object_encryption, dest_object_checksum_algorithm) = match dest_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), + _ => unreachable!(), + }; + let (dest_encryption, _) = + EncryptionParams::check_decrypt(&garage, req.headers(), &dest_object_encryption)?; + let same_encryption = EncryptionParams::is_same(&source_encryption, &dest_encryption); // Check source range is valid let source_range = match req.headers().get("x-amz-copy-source-range") { @@ -217,53 +413,30 @@ pub async fn handle_upload_part_copy( }, }; - // Check destination version is indeed in uploading state - if !dest_object - .versions() - .iter() - .any(|v| v.uuid == dest_version_uuid && v.is_uploading()) - { - return Err(Error::NoSuchUpload); - } - // Check source version is not inlined - match source_version_data { - ObjectVersionData::DeleteMarker => unreachable!(), - ObjectVersionData::Inline(_meta, _bytes) => { - // This is only for small files, we don't bother handling this. - // (in AWS UploadPartCopy works for parts at least 5MB which - // is never the case of an inline object) - return Err(Error::bad_request( - "Source object is too small (minimum part size is 5Mb)", - )); - } - ObjectVersionData::FirstBlock(_meta, _first_block_hash) => (), - }; - - // Fetch source versin with its block list, - // and destination version to check part hasn't yet been uploaded - let (source_version, dest_version) = futures::try_join!( - garage - .version_table - .get(&source_object_version.uuid, &EmptyKey), - garage.version_table.get(&dest_version_uuid, &EmptyKey), - )?; - let source_version = source_version.ok_or(Error::NoSuchKey)?; - - // Check this part number hasn't yet been uploaded - if let Some(dv) = dest_version { - if dv.has_part_number(part_number) { - return Err(Error::bad_request(format!( - "Part number {} has already been uploaded", - part_number - ))); - } + if matches!(source_version_data, ObjectVersionData::Inline(_, _)) { + // This is only for small files, we don't bother handling this. + // (in AWS UploadPartCopy works for parts at least 5MB which + // is never the case of an inline object) + return Err(Error::bad_request( + "Source object is too small (minimum part size is 5Mb)", + )); } + // Fetch source version with its block list + let source_version = garage + .version_table + .get(&source_object_version.uuid, &EmptyKey) + .await? + .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; + // We want to reuse blocks from the source version as much as possible. // However, we still need to get the data from these blocks // because we need to know it to calculate the MD5sum of the part - // which is used as its ETag. + // which is used as its ETag. For encrypted sources or destinations, + // we must always read(+decrypt) and then write(+encrypt), so we + // can never reuse data blocks as is. // First, calculate what blocks we want to keep, // and the subrange of the block to take, if the bounds of the @@ -299,31 +472,83 @@ pub async fn handle_upload_part_copy( current_offset = block_end; } + // Calculate the identity of destination part: timestamp, version id + let dest_version_id = gen_uuid(); + let dest_mpu_part_key = MpuPartKey { + part_number, + timestamp: dest_mpu.next_timestamp(part_number), + }; + + // Create the uploaded part + dest_mpu.parts.clear(); + dest_mpu.parts.put( + dest_mpu_part_key, + MpuPart { + version: dest_version_id, + // These are all filled in later (bottom of this function) + etag: None, + checksum: None, + size: None, + }, + ); + garage.mpu_table.insert(&dest_mpu).await?; + + let mut dest_version = Version::new( + dest_version_id, + VersionBacklink::MultipartUpload { + upload_id: dest_upload_id, + }, + false, + ); + // write an empty version now to be the parent of the block_ref entries + garage.version_table.insert(&dest_version).await?; + // Now, actually copy the blocks - let mut md5hasher = Md5::new(); + let mut checksummer = Checksummer::init(&Default::default(), !dest_encryption.is_encrypted()) + .add(dest_object_checksum_algorithm); // First, create a stream that is able to read the source blocks // and extract the subrange if necessary. // The second returned value is an Option, that is Some // if and only if the block returned is a block that already existed - // in the Garage data store (thus we don't need to save it again). + // in the Garage data store and can be reused as-is instead of having + // to save it again. This excludes encrypted source blocks that we had + // to decrypt. let garage2 = garage.clone(); let order_stream = OrderTag::stream(); let source_blocks = stream::iter(blocks_to_copy) .enumerate() - .flat_map(|(i, (block_hash, range_to_copy))| { + .map(|(i, (block_hash, range_to_copy))| { let garage3 = garage2.clone(); - stream::once(async move { - let data = garage3 - .block_manager - .rpc_get_block(&block_hash, Some(order_stream.order(i as u64))) + async move { + let stream = source_encryption + .get_block(&garage3, &block_hash, Some(order_stream.order(i as u64))) .await?; + let data = read_stream_to_end(stream).await?.into_bytes(); + // For each item, we return a tuple of: + // 1. the full data block (decrypted) + // 2. an Option that indicates the hash of the block in the block store, + // only if it can be re-used as-is in the copied object match range_to_copy { - Some(r) => Ok((data.slice(r), None)), - None => Ok((data, Some(block_hash))), + Some(r) => { + // If we are taking a subslice of the data, we cannot reuse the block as-is + Ok((data.slice(r), None)) + } + None if same_encryption => { + // If the data is unencrypted before & after, or if we are using + // the same encryption key, we can reuse the stored block, no need + // to re-send it to storage nodes. + Ok((data, Some(block_hash))) + } + None => { + // If we are decrypting / (re)encrypting with different keys, + // we cannot reuse the block as-is + Ok((data, None)) + } } - }) + } }) + .buffered(2) .peekable(); // The defragmenter is a custom stream (defined below) that concatenates @@ -331,70 +556,118 @@ pub async fn handle_upload_part_copy( // It returns a series of (Vec, Option). // When it is done, it returns an empty vec. // Same as the previous iterator, the Option is Some(_) if and only if - // it's an existing block of the Garage data store. + // it's an existing block of the Garage data store that can be reused. let mut defragmenter = Defragmenter::new(garage.config.block_size, Box::pin(source_blocks)); let mut current_offset = 0; let mut next_block = defragmenter.next().await?; + let mut blocks_to_dup = dest_version.clone(); + // TODO this could be optimized similarly to read_and_put_blocks + // low priority because uploadpartcopy is rarely used loop { let (data, existing_block_hash) = next_block; if data.is_empty() { break; } - md5hasher.update(&data[..]); + let data_len = data.len() as u64; - let must_upload = existing_block_hash.is_none(); - let final_hash = existing_block_hash.unwrap_or_else(|| blake2sum(&data[..])); + let (checksummer_updated, (data_to_upload, final_hash)) = + tokio::task::spawn_blocking(move || { + checksummer.update(&data[..]); - let mut version = Version::new(dest_version_uuid, dest_bucket_id, dest_key.clone(), false); - version.blocks.put( + let tup = match existing_block_hash { + Some(hash) if same_encryption => (None, hash), + _ => { + let data_enc = dest_encryption.encrypt_block(data)?; + let hash = blake2sum(&data_enc); + (Some(data_enc), hash) + } + }; + Ok::<_, Error>((checksummer, tup)) + }) + .await + .unwrap()?; + checksummer = checksummer_updated; + + let (version_block_key, version_block) = ( VersionBlockKey { part_number, offset: current_offset, }, VersionBlock { hash: final_hash, - size: data.len() as u64, + size: data_len, }, ); - current_offset += data.len() as u64; + current_offset += data_len; - let block_ref = BlockRef { - block: final_hash, - version: dest_version_uuid, - deleted: false.into(), + let next = if let Some(final_data) = data_to_upload { + dest_version.blocks.clear(); + dest_version.blocks.put(version_block_key, version_block); + let block_ref = BlockRef { + block: final_hash, + version: dest_version_id, + deleted: false.into(), + }; + let (_, _, _, next) = futures::try_join!( + // Thing 1: if the block is not exactly a block that existed before, + // we need to insert that data as a new block. + garage.block_manager.rpc_put_block( + final_hash, + final_data, + dest_encryption.is_encrypted(), + None + ), + // Thing 2: we need to insert the block in the version + garage.version_table.insert(&dest_version), + // Thing 3: we need to add a block reference + garage.block_ref_table.insert(&block_ref), + // Thing 4: we need to read the next block + defragmenter.next(), + )?; + next + } else { + blocks_to_dup.blocks.put(version_block_key, version_block); + defragmenter.next().await? }; - - let garage2 = garage.clone(); - let res = futures::try_join!( - // Thing 1: if the block is not exactly a block that existed before, - // we need to insert that data as a new block. - async move { - if must_upload { - garage2.block_manager.rpc_put_block(final_hash, data).await - } else { - Ok(()) - } - }, - // Thing 2: we need to insert the block in the version - garage.version_table.insert(&version), - // Thing 3: we need to add a block reference - garage.block_ref_table.insert(&block_ref), - // Thing 4: we need to prefetch the next block - defragmenter.next(), - )?; - next_block = res.3; + next_block = next; } - let data_md5sum = md5hasher.finalize(); - let etag = hex::encode(data_md5sum); + assert_eq!(current_offset, source_range.length); + + // Put the duplicated blocks into the version & block_refs tables + let block_refs_to_put = blocks_to_dup + .blocks + .items() + .iter() + .map(|b| BlockRef { + block: b.1.hash, + version: dest_version_id, + deleted: false.into(), + }) + .collect::>(); + futures::try_join!( + garage.version_table.insert(&blocks_to_dup), + garage.block_ref_table.insert_many(&block_refs_to_put[..]), + )?; + + let checksums = checksummer.finalize(); + let etag = dest_encryption.etag_from_md5(&checksums.md5); + let checksum = checksums.extract(dest_object_checksum_algorithm); // Put the part's ETag in the Versiontable - let mut version = Version::new(dest_version_uuid, dest_bucket_id, dest_key.clone(), false); - version.parts_etags.put(part_number, etag.clone()); - garage.version_table.insert(&version).await?; + dest_mpu.parts.put( + dest_mpu_part_key, + MpuPart { + version: dest_version_id, + etag: Some(etag.clone()), + checksum, + size: Some(current_offset), + }, + ); + garage.mpu_table.insert(&dest_mpu).await?; // LGTM let resp_xml = s3_xml::to_xml_with_header(&CopyPartResult { @@ -403,20 +676,21 @@ pub async fn handle_upload_part_copy( last_modified: s3_xml::Value(msec_to_rfc3339(source_object_version.timestamp)), })?; - Ok(Response::builder() + let mut resp = Response::builder() .header("Content-Type", "application/xml") .header( "x-amz-copy-source-version-id", hex::encode(source_object_version.uuid), - ) - .body(Body::from(resp_xml))?) + ); + dest_encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(resp_xml))?) } -async fn get_copy_source( - garage: &Garage, - api_key: &Key, - req: &Request, -) -> Result { +async fn get_copy_source(ctx: &ReqCtx, req: &Request) -> Result { + let ReqCtx { + garage, api_key, .. + } = ctx; + let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?; @@ -424,7 +698,8 @@ async fn get_copy_source( let source_bucket_id = garage .bucket_helper() .resolve_bucket(&source_bucket.to_string(), api_key) - .await?; + .await + .map_err(pass_helper_error)?; if !api_key.allow_read(&source_bucket_id) { return Err(Error::forbidden(format!( @@ -470,97 +745,6 @@ fn extract_source_info( Ok((source_version, source_version_data, source_version_meta)) } -struct CopyPreconditionHeaders { - copy_source_if_match: Option>, - copy_source_if_modified_since: Option, - copy_source_if_none_match: Option>, - copy_source_if_unmodified_since: Option, -} - -impl CopyPreconditionHeaders { - fn parse(req: &Request) -> Result { - Ok(Self { - copy_source_if_match: req - .headers() - .get("x-amz-copy-source-if-match") - .map(|x| x.to_str()) - .transpose()? - .map(|x| { - x.split(',') - .map(|m| m.trim().trim_matches('"').to_string()) - .collect::>() - }), - copy_source_if_modified_since: req - .headers() - .get("x-amz-copy-source-if-modified-since") - .map(|x| x.to_str()) - .transpose()? - .map(httpdate::parse_http_date) - .transpose() - .ok_or_bad_request("Invalid date in x-amz-copy-source-if-modified-since")?, - copy_source_if_none_match: req - .headers() - .get("x-amz-copy-source-if-none-match") - .map(|x| x.to_str()) - .transpose()? - .map(|x| { - x.split(',') - .map(|m| m.trim().trim_matches('"').to_string()) - .collect::>() - }), - copy_source_if_unmodified_since: req - .headers() - .get("x-amz-copy-source-if-unmodified-since") - .map(|x| x.to_str()) - .transpose()? - .map(httpdate::parse_http_date) - .transpose() - .ok_or_bad_request("Invalid date in x-amz-copy-source-if-unmodified-since")?, - }) - } - - fn check(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> { - let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp); - - let ok = match ( - &self.copy_source_if_match, - &self.copy_source_if_unmodified_since, - &self.copy_source_if_none_match, - &self.copy_source_if_modified_since, - ) { - // TODO I'm not sure all of the conditions are evaluated correctly here - - // If we have both if-match and if-unmodified-since, - // basically we don't care about if-unmodified-since, - // because in the spec it says that if if-match evaluates to - // true but if-unmodified-since evaluates to false, - // the copy is still done. - (Some(im), _, None, None) => im.iter().any(|x| x == etag || x == "*"), - (None, Some(ius), None, None) => v_date <= *ius, - - // If we have both if-none-match and if-modified-since, - // then both of the two conditions must evaluate to true - (None, None, Some(inm), Some(ims)) => { - !inm.iter().any(|x| x == etag || x == "*") && v_date > *ims - } - (None, None, Some(inm), None) => !inm.iter().any(|x| x == etag || x == "*"), - (None, None, None, Some(ims)) => v_date > *ims, - (None, None, None, None) => true, - _ => { - return Err(Error::bad_request( - "Invalid combination of x-amz-copy-source-if-xxxxx headers", - )) - } - }; - - if ok { - Ok(()) - } else { - Err(Error::PreconditionFailed) - } - } -} - type BlockStreamItemOk = (Bytes, Option); type BlockStreamItem = Result; @@ -630,7 +814,7 @@ pub struct CopyPartResult { #[cfg(test)] mod tests { use super::*; - use crate::s3::xml::to_xml_with_header; + use crate::xml::to_xml_with_header; #[test] fn copy_object_result() -> Result<(), Error> { diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index c7273464..1f365beb 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -1,28 +1,20 @@ use quick_xml::de::from_reader; -use std::sync::Arc; -use http::header::{ - ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, - ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, -}; -use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode}; +use hyper::{header::HeaderName, Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; - use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; -use garage_util::data::*; -pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +use garage_api_common::helpers::*; - if let Some(cors) = param.cors_config.get() { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; + +pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(cors) = bucket_params.cors_config.get() { let wc = CorsConfiguration { xmlns: (), cors_rules: cors @@ -34,208 +26,59 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } else { Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } } -pub async fn handle_delete_cors( - garage: Arc, - bucket_id: Uuid, -) -> Result, Error> { - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) +pub async fn handle_delete_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.cors_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) .await?; - let param = bucket.params_mut().unwrap(); - - param.cors_config.update(None); - garage.bucket_table.insert(&bucket).await?; - Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_put_cors( - garage: Arc, - bucket_id: Uuid, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - - let param = bucket.params_mut().unwrap(); + let body = req.into_body().collect().await?; let conf: CorsConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .cors_config .update(Some(conf.into_garage_cors_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) - .body(Body::empty())?) -} - -pub async fn handle_options_s3api( - garage: Arc, - req: &Request, - bucket_name: Option, -) -> Result, Error> { - // FIXME: CORS rules of buckets with local aliases are - // not taken into account. - - // If the bucket name is a global bucket name, - // we try to apply the CORS rules of that bucket. - // If a user has a local bucket name that has - // the same name, its CORS rules won't be applied - // and will be shadowed by the rules of the globally - // existing bucket (but this is inevitable because - // OPTIONS calls are not auhtenticated). - if let Some(bn) = bucket_name { - let helper = garage.bucket_helper(); - let bucket_id = helper.resolve_global_bucket_name(&bn).await?; - if let Some(id) = bucket_id { - let bucket = garage.bucket_helper().get_existing_bucket(id).await?; - handle_options_for_bucket(req, &bucket) - } else { - // If there is a bucket name in the request, but that name - // does not correspond to a global alias for a bucket, - // then it's either a non-existing bucket or a local bucket. - // We have no way of knowing, because the request is not - // authenticated and thus we can't resolve local aliases. - // We take the permissive approach of allowing everything, - // because we don't want to prevent web apps that use - // local bucket names from making API calls. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "*") - .status(StatusCode::OK) - .body(Body::empty())?) - } - } else { - // If there is no bucket name in the request, - // we are doing a ListBuckets call, which we want to allow - // for all origins. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") - .status(StatusCode::OK) - .body(Body::empty())?) - } -} - -pub fn handle_options_for_bucket( - req: &Request, - bucket: &Bucket, -) -> Result, Error> { - let origin = req - .headers() - .get("Origin") - .ok_or_bad_request("Missing Origin header")? - .to_str()?; - let request_method = req - .headers() - .get(ACCESS_CONTROL_REQUEST_METHOD) - .ok_or_bad_request("Missing Access-Control-Request-Method header")? - .to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { - let matching_rule = cors_config - .iter() - .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); - if let Some(rule) = matching_rule { - let mut resp = Response::builder() - .status(StatusCode::OK) - .body(Body::empty())?; - add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; - return Ok(resp); - } - } - - Err(Error::forbidden("This CORS request is not allowed.")) -} - -pub fn find_matching_cors_rule<'a>( - bucket: &'a Bucket, - req: &Request, -) -> Result, Error> { - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { - if let Some(origin) = req.headers().get("Origin") { - let origin = origin.to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - return Ok(cors_config.iter().find(|rule| { - cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) - })); - } - } - Ok(None) -} - -fn cors_rule_matches<'a, HI, S>( - rule: &GarageCorsRule, - origin: &'a str, - method: &'a str, - mut request_headers: HI, -) -> bool -where - HI: Iterator, - S: AsRef, -{ - rule.allow_origins.iter().any(|x| x == "*" || x == origin) - && rule.allow_methods.iter().any(|x| x == "*" || x == method) - && request_headers.all(|h| { - rule.allow_headers - .iter() - .any(|x| x == "*" || x == h.as_ref()) - }) -} - -pub fn add_cors_headers( - resp: &mut Response, - rule: &GarageCorsRule, -) -> Result<(), http::header::InvalidHeaderValue> { - let h = resp.headers_mut(); - h.insert( - ACCESS_CONTROL_ALLOW_ORIGIN, - rule.allow_origins.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_METHODS, - rule.allow_methods.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_HEADERS, - rule.allow_headers.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_EXPOSE_HEADERS, - rule.expose_headers.join(", ").parse()?, - ); - Ok(()) + .body(empty_body())?) } // ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- @@ -245,7 +88,9 @@ pub fn add_cors_headers( pub struct CorsConfiguration { #[serde(serialize_with = "xmlns_tag", skip_deserializing)] pub xmlns: (), - #[serde(rename = "CORSRule")] + // "default" is required to be able to parse an empty list of rules, + // cf https://docs.rs/quick-xml/latest/quick_xml/de/#sequences-xsall-and-xssequence-xml-schema-types + #[serde(rename = "CORSRule", default)] pub cors_rules: Vec, } @@ -427,4 +272,26 @@ mod tests { Ok(()) } + + #[test] + fn test_deserialize_norules() -> Result<(), Error> { + let message = r#" +"#; + let conf: CorsConfiguration = from_str(message).unwrap(); + let ref_value = CorsConfiguration { + xmlns: (), + cors_rules: vec![], + }; + assert_eq! { + ref_value, + conf + }; + + let message2 = to_xml_with_header(&ref_value)?; + + let cleanup = |c: &str| c.replace(char::is_whitespace, ""); + assert_eq!(cleanup(message), cleanup(&message2)); + + Ok(()) + } } diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index b337155f..d785b9d8 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -1,89 +1,73 @@ -use std::sync::Arc; - -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use garage_util::data::*; -use garage_util::time::*; -use garage_model::garage::Garage; use garage_model::s3::object_table::*; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::helpers::*; -async fn handle_delete_internal( - garage: &Garage, - bucket_id: Uuid, - key: &str, -) -> Result<(Uuid, Uuid), Error> { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::put::next_timestamp; +use crate::xml as s3_xml; + +async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let object = garage .object_table - .get(&bucket_id, &key.to_string()) + .get(bucket_id, &key.to_string()) .await? .ok_or(Error::NoSuchKey)?; // No need to delete - let interesting_versions = object.versions().iter().filter(|v| { - !matches!( - v.state, - ObjectVersionState::Aborted - | ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) - ) - }); + let del_timestamp = next_timestamp(Some(&object)); + let del_uuid = gen_uuid(); - let mut version_to_delete = None; - let mut timestamp = now_msec(); - for v in interesting_versions { - if v.timestamp + 1 > timestamp || version_to_delete.is_none() { - version_to_delete = Some(v.uuid); + let deleted_version = object + .versions() + .iter() + .rev() + .find(|v| !matches!(&v.state, ObjectVersionState::Aborted)) + .or_else(|| object.versions().iter().rev().next()); + let deleted_version = match deleted_version { + Some(dv) => dv.uuid, + None => { + warn!("Object has no versions: {:?}", object); + Uuid::from([0u8; 32]) } - timestamp = std::cmp::max(timestamp, v.timestamp + 1); - } - - let deleted_version = version_to_delete.ok_or(Error::NoSuchKey)?; - - let version_uuid = gen_uuid(); + }; let object = Object::new( - bucket_id, + *bucket_id, key.into(), vec![ObjectVersion { - uuid: version_uuid, - timestamp, + uuid: del_uuid, + timestamp: del_timestamp, state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker), }], ); garage.object_table.insert(&object).await?; - Ok((deleted_version, version_uuid)) + Ok((deleted_version, del_uuid)) } -pub async fn handle_delete( - garage: Arc, - bucket_id: Uuid, - key: &str, -) -> Result, Error> { - match handle_delete_internal(&garage, bucket_id, key).await { +pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result, Error> { + match handle_delete_internal(&ctx, key).await { Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::from(vec![])) + .body(empty_body()) .unwrap()), Err(e) => Err(e), } } pub async fn handle_delete_objects( - garage: Arc, - bucket_id: Uuid, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let body = req.into_body().collect().await?; let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?; let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?; @@ -92,7 +76,7 @@ pub async fn handle_delete_objects( let mut ret_errors = Vec::new(); for obj in cmd.objects.iter() { - match handle_delete_internal(&garage, bucket_id, &obj.key).await { + match handle_delete_internal(&ctx, &obj.key).await { Ok((deleted_version, delete_marker_version)) => { if cmd.quiet { continue; @@ -122,7 +106,7 @@ pub async fn handle_delete_objects( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } struct DeleteRequest { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs new file mode 100644 index 00000000..fa7285ca --- /dev/null +++ b/src/api/s3/encryption.rs @@ -0,0 +1,596 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::pin::Pin; + +use aes_gcm::{ + aead::stream::{DecryptorLE31, EncryptorLE31, StreamLE31}, + aead::{Aead, AeadCore, KeyInit, OsRng}, + aes::cipher::crypto_common::rand_core::RngCore, + aes::cipher::typenum::Unsigned, + Aes256Gcm, Key, Nonce, +}; +use base64::prelude::*; +use bytes::Bytes; + +use futures::stream::Stream; +use futures::task; +use tokio::io::BufReader; + +use http::header::{HeaderMap, HeaderName, HeaderValue}; + +use garage_net::bytes_buf::BytesBuf; +use garage_net::stream::{stream_asyncread, ByteStream}; +use garage_rpc::rpc_helper::OrderTag; +use garage_util::data::Hash; +use garage_util::error::Error as GarageError; +use garage_util::migrate::Migrate; + +use garage_model::garage::Garage; +use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; + +use garage_api_common::common_error::*; +use garage_api_common::signature::checksum::Md5Checksum; + +use crate::error::Error; + +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-key"); +const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName = + HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5"); + +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-algorithm"); +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-key"); +const X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName = + HeaderName::from_static("x-amz-copy-source-server-side-encryption-customer-key-md5"); + +const CUSTOMER_ALGORITHM_AES256: &[u8] = b"AES256"; + +type Md5Output = md5::digest::Output; + +type StreamNonceSize = aes_gcm::aead::stream::NonceSize>; + +// Data blocks are encrypted by smaller chunks of size 4096 bytes, +// so that data can be streamed when reading. +// This size has to be known and has to be constant, or data won't be +// readable anymore. DO NOT CHANGE THIS VALUE. +const STREAM_ENC_PLAIN_CHUNK_SIZE: usize = 0x1000; // 4096 bytes +const STREAM_ENC_CYPER_CHUNK_SIZE: usize = STREAM_ENC_PLAIN_CHUNK_SIZE + 16; + +#[derive(Clone, Copy)] +pub enum EncryptionParams { + Plaintext, + SseC { + client_key: Key, + client_key_md5: Md5Output, + compression_level: Option, + }, +} + +impl EncryptionParams { + pub fn is_encrypted(&self) -> bool { + !matches!(self, Self::Plaintext) + } + + pub fn is_same(a: &Self, b: &Self) -> bool { + let relevant_info = |x: &Self| match x { + Self::Plaintext => None, + Self::SseC { + client_key, + compression_level, + .. + } => Some((*client_key, compression_level.is_some())), + }; + relevant_info(a) == relevant_info(b) + } + + pub fn new_from_headers( + garage: &Garage, + headers: &HeaderMap, + ) -> Result { + let key = parse_request_headers( + headers, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + match key { + Some((client_key, client_key_md5)) => Ok(EncryptionParams::SseC { + client_key, + client_key_md5, + compression_level: garage.config.compression_level, + }), + None => Ok(EncryptionParams::Plaintext), + } + } + + pub fn add_response_headers(&self, resp: &mut http::response::Builder) { + if let Self::SseC { client_key_md5, .. } = self { + let md5 = BASE64_STANDARD.encode(&client_key_md5); + + resp.headers_mut().unwrap().insert( + X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + HeaderValue::from_bytes(CUSTOMER_ALGORITHM_AES256).unwrap(), + ); + resp.headers_mut().unwrap().insert( + X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + HeaderValue::from_bytes(md5.as_bytes()).unwrap(), + ); + } + } + + pub fn check_decrypt<'a>( + garage: &Garage, + headers: &HeaderMap, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + let key = parse_request_headers( + headers, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + Self::check_decrypt_common(garage, key, obj_enc) + } + + pub fn check_decrypt_for_copy_source<'a>( + garage: &Garage, + headers: &HeaderMap, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + let key = parse_request_headers( + headers, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + &X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + )?; + Self::check_decrypt_common(garage, key, obj_enc) + } + + fn check_decrypt_common<'a>( + garage: &Garage, + key: Option<(Key, Md5Output)>, + obj_enc: &'a ObjectVersionEncryption, + ) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> { + match (key, &obj_enc) { + ( + Some((client_key, client_key_md5)), + ObjectVersionEncryption::SseC { inner, compressed }, + ) => { + let enc = Self::SseC { + client_key, + client_key_md5, + compression_level: if *compressed { + Some(garage.config.compression_level.unwrap_or(1)) + } else { + None + }, + }; + let plaintext = enc.decrypt_blob(&inner)?; + let inner = ObjectVersionMetaInner::decode(&plaintext) + .ok_or_internal_error("Could not decode encrypted metadata")?; + Ok((enc, Cow::Owned(inner))) + } + (None, ObjectVersionEncryption::Plaintext { inner }) => { + Ok((Self::Plaintext, Cow::Borrowed(inner))) + } + (_, ObjectVersionEncryption::SseC { .. }) => { + Err(Error::bad_request("Object is encrypted")) + } + (Some(_), _) => { + // TODO: should this be an OK scenario? + Err(Error::bad_request("Trying to decrypt a plaintext object")) + } + } + } + + pub fn encrypt_meta( + &self, + meta: ObjectVersionMetaInner, + ) -> Result { + match self { + Self::SseC { + compression_level, .. + } => { + let plaintext = meta.encode().map_err(GarageError::from)?; + let ciphertext = self.encrypt_blob(&plaintext)?; + Ok(ObjectVersionEncryption::SseC { + inner: ciphertext.into_owned(), + compressed: compression_level.is_some(), + }) + } + Self::Plaintext => Ok(ObjectVersionEncryption::Plaintext { inner: meta }), + } + } + + // ---- generating object Etag values ---- + pub fn etag_from_md5(&self, md5sum: &Option) -> String { + match self { + Self::Plaintext => md5sum + .map(|x| hex::encode(&x[..])) + .expect("md5 digest should have been computed"), + Self::SseC { .. } => { + // AWS specifies that for encrypted objects, the Etag is not + // the md5sum of the data, but doesn't say what it is. + // So we just put some random bytes. + let mut random = [0u8; 16]; + OsRng.fill_bytes(&mut random); + hex::encode(&random) + } + } + } + + // ---- generic function for encrypting / decrypting blobs ---- + // Prepends a randomly-generated nonce to the encrypted value. + // This is used for encrypting object metadata and inlined data for small objects. + // This does not compress anything. + + pub fn encrypt_blob<'a>(&self, blob: &'a [u8]) -> Result, Error> { + match self { + Self::SseC { client_key, .. } => { + let cipher = Aes256Gcm::new(&client_key); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ciphertext = cipher + .encrypt(&nonce, blob) + .ok_or_internal_error("Encryption failed")?; + Ok(Cow::Owned([nonce.to_vec(), ciphertext].concat())) + } + Self::Plaintext => Ok(Cow::Borrowed(blob)), + } + } + + pub fn decrypt_blob<'a>(&self, blob: &'a [u8]) -> Result, Error> { + match self { + Self::SseC { client_key, .. } => { + let cipher = Aes256Gcm::new(&client_key); + let nonce_size = ::NonceSize::to_usize(); + let nonce = Nonce::from_slice( + blob.get(..nonce_size) + .ok_or_internal_error("invalid encrypted data")?, + ); + let plaintext = cipher + .decrypt(nonce, &blob[nonce_size..]) + .ok_or_bad_request( + "Invalid encryption key, could not decrypt object metadata.", + )?; + Ok(Cow::Owned(plaintext)) + } + Self::Plaintext => Ok(Cow::Borrowed(blob)), + } + } + + // ---- function for encrypting / decrypting byte streams ---- + + /// Get a data block from the storage node, and decrypt+decompress it + /// if necessary. If object is plaintext, just get it without any processing. + pub async fn get_block( + &self, + garage: &Garage, + hash: &Hash, + order: Option, + ) -> Result { + let raw_block = garage + .block_manager + .rpc_get_block_streaming(hash, order) + .await?; + Ok(self.decrypt_block_stream(raw_block)) + } + + pub fn decrypt_block_stream(&self, stream: ByteStream) -> ByteStream { + match self { + Self::Plaintext => stream, + Self::SseC { + client_key, + compression_level, + .. + } => { + let plaintext = DecryptStream::new(stream, *client_key); + if compression_level.is_some() { + let reader = stream_asyncread(Box::pin(plaintext)); + let reader = BufReader::new(reader); + let reader = async_compression::tokio::bufread::ZstdDecoder::new(reader); + Box::pin(tokio_util::io::ReaderStream::new(reader)) + } else { + Box::pin(plaintext) + } + } + } + } + + /// Encrypt a data block if encryption is set, for use before + /// putting the data blocks into storage + pub fn encrypt_block(&self, block: Bytes) -> Result { + match self { + Self::Plaintext => Ok(block), + Self::SseC { + client_key, + compression_level, + .. + } => { + let block = if let Some(level) = compression_level { + Cow::Owned( + garage_block::zstd_encode(block.as_ref(), *level) + .ok_or_internal_error("failed to compress data block")?, + ) + } else { + Cow::Borrowed(block.as_ref()) + }; + + let mut ret = Vec::with_capacity(block.len() + 32 + block.len() / 64); + + let mut nonce: Nonce = Default::default(); + OsRng.fill_bytes(&mut nonce); + ret.extend_from_slice(nonce.as_slice()); + + let mut cipher = EncryptorLE31::::new(&client_key, &nonce); + let mut iter = block.chunks(STREAM_ENC_PLAIN_CHUNK_SIZE).peekable(); + + if iter.peek().is_none() { + // Empty stream: we encrypt an empty last chunk + let chunk_enc = cipher + .encrypt_last(&[][..]) + .ok_or_internal_error("failed to encrypt chunk")?; + ret.extend_from_slice(&chunk_enc); + } else { + loop { + let chunk = iter.next().unwrap(); + if iter.peek().is_some() { + let chunk_enc = cipher + .encrypt_next(chunk) + .ok_or_internal_error("failed to encrypt chunk")?; + assert_eq!(chunk.len(), STREAM_ENC_PLAIN_CHUNK_SIZE); + assert_eq!(chunk_enc.len(), STREAM_ENC_CYPER_CHUNK_SIZE); + ret.extend_from_slice(&chunk_enc); + } else { + // use encrypt_last for the last chunk + let chunk_enc = cipher + .encrypt_last(chunk) + .ok_or_internal_error("failed to encrypt chunk")?; + ret.extend_from_slice(&chunk_enc); + break; + } + } + } + + Ok(ret.into()) + } + } + } +} + +fn parse_request_headers( + headers: &HeaderMap, + alg_header: &HeaderName, + key_header: &HeaderName, + md5_header: &HeaderName, +) -> Result, Md5Output)>, Error> { + let alg = headers.get(alg_header).map(HeaderValue::as_bytes); + let key = headers.get(key_header).map(HeaderValue::as_bytes); + let md5 = headers.get(md5_header).map(HeaderValue::as_bytes); + + match alg { + Some(CUSTOMER_ALGORITHM_AES256) => { + use md5::{Digest, Md5}; + + let key_b64 = + key.ok_or_bad_request("Missing server-side-encryption-customer-key header")?; + let key_bytes: [u8; 32] = BASE64_STANDARD + .decode(&key_b64) + .ok_or_bad_request( + "Invalid server-side-encryption-customer-key header: invalid base64", + )? + .try_into() + .ok() + .ok_or_bad_request( + "Invalid server-side-encryption-customer-key header: invalid length", + )?; + + let md5_b64 = + md5.ok_or_bad_request("Missing server-side-encryption-customer-key-md5 header")?; + let md5_bytes = BASE64_STANDARD.decode(&md5_b64).ok_or_bad_request( + "Invalid server-side-encryption-customer-key-md5 header: invalid bass64", + )?; + + let mut hasher = Md5::new(); + hasher.update(&key_bytes[..]); + let our_md5 = hasher.finalize(); + if our_md5.as_slice() != md5_bytes.as_slice() { + return Err(Error::bad_request( + "Server-side encryption client key MD5 checksum does not match", + )); + } + + Ok(Some((key_bytes.into(), our_md5))) + } + Some(alg) => Err(Error::InvalidEncryptionAlgorithm( + String::from_utf8_lossy(alg).into_owned(), + )), + None => { + if key.is_some() || md5.is_some() { + Err(Error::bad_request( + "Unexpected server-side-encryption-customer-key{,-md5} header(s)", + )) + } else { + Ok(None) + } + } + } +} + +// ---- encrypt & decrypt streams ---- + +#[pin_project::pin_project] +struct DecryptStream { + #[pin] + stream: ByteStream, + done_reading: bool, + buf: BytesBuf, + key: Key, + state: DecryptStreamState, +} + +enum DecryptStreamState { + Starting, + Running(DecryptorLE31), + Done, +} + +impl DecryptStream { + fn new(stream: ByteStream, key: Key) -> Self { + Self { + stream, + done_reading: false, + buf: BytesBuf::new(), + key, + state: DecryptStreamState::Starting, + } + } +} + +impl Stream for DecryptStream { + type Item = Result; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll> { + use std::task::Poll; + + let mut this = self.project(); + + // The first bytes of the stream should contain the starting nonce. + // If we don't have a Running state, it means that we haven't + // yet read the nonce. + while matches!(this.state, DecryptStreamState::Starting) { + let nonce_size = StreamNonceSize::to_usize(); + if let Some(nonce) = this.buf.take_exact(nonce_size) { + let nonce = Nonce::from_slice(nonce.as_ref()); + *this.state = DecryptStreamState::Running(DecryptorLE31::new(&this.key, nonce)); + break; + } + + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(e))); + } + None => { + return Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "Decrypt: unexpected EOF, could not read nonce", + )))); + } + } + } + + // Read at least one byte more than the encrypted chunk size + // (if possible), so that we know if we are decrypting the + // last chunk or not. + while !*this.done_reading && this.buf.len() <= STREAM_ENC_CYPER_CHUNK_SIZE { + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(e))); + } + None => { + *this.done_reading = true; + break; + } + } + } + + if matches!(this.state, DecryptStreamState::Done) { + if !this.buf.is_empty() { + return Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Decrypt: unexpected bytes after last encrypted chunk", + )))); + } + return Poll::Ready(None); + } + + let res = if this.buf.len() > STREAM_ENC_CYPER_CHUNK_SIZE { + // we have strictly more bytes than the encrypted chunk size, + // so we know this is not the last + let DecryptStreamState::Running(ref mut cipher) = this.state else { + unreachable!() + }; + let chunk = this.buf.take_exact(STREAM_ENC_CYPER_CHUNK_SIZE).unwrap(); + let chunk_dec = cipher.decrypt_next(chunk.as_ref()); + if let Ok(c) = &chunk_dec { + assert_eq!(c.len(), STREAM_ENC_PLAIN_CHUNK_SIZE); + } + chunk_dec + } else { + // We have one encrypted chunk size or less, even though we tried + // to read more, so this is the last chunk. Decrypt using the + // appropriate decrypt_last() function that then destroys the cipher. + let state = std::mem::replace(this.state, DecryptStreamState::Done); + let DecryptStreamState::Running(cipher) = state else { + unreachable!() + }; + let chunk = this.buf.take_all(); + cipher.decrypt_last(chunk.as_ref()) + }; + + match res { + Ok(bytes) if bytes.is_empty() => Poll::Ready(None), + Ok(bytes) => Poll::Ready(Some(Ok(bytes.into()))), + Err(_) => Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Decryption failed", + )))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use futures::stream::StreamExt; + use garage_net::stream::read_stream_to_end; + + fn stream() -> ByteStream { + Box::pin( + futures::stream::iter(16usize..1024) + .map(|i| Ok(Bytes::from(vec![(i % 256) as u8; (i * 37) % 1024]))), + ) + } + + async fn test_block_enc(compression_level: Option) { + let enc = EncryptionParams::SseC { + client_key: Aes256Gcm::generate_key(&mut OsRng), + client_key_md5: Default::default(), // not needed + compression_level, + }; + + let block_plain = read_stream_to_end(stream()).await.unwrap().into_bytes(); + + let block_enc = enc.encrypt_block(block_plain.clone()).unwrap(); + + let block_dec = + enc.decrypt_block_stream(Box::pin(futures::stream::once(async { Ok(block_enc) }))); + let block_dec = read_stream_to_end(block_dec).await.unwrap().into_bytes(); + + assert_eq!(block_plain, block_dec); + assert!(block_dec.len() > 128000); + } + + #[tokio::test] + async fn test_encrypt_block() { + test_block_enc(None).await + } + + #[tokio::test] + async fn test_encrypt_block_compressed() { + test_block_enc(Some(1)).await + } +} diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index c50cff9f..64112084 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -1,100 +1,107 @@ use std::convert::TryInto; -use err_derive::Error; use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; use garage_model::helper::error::Error as HelperError; -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::s3::xml as s3_xml; -use crate::signature::error::Error as SignatureError; +pub(crate) use garage_api_common::common_error::pass_helper_error; + +use garage_api_common::common_error::{ + commonErrorDerivative, helper_error_as_internal, CommonError, +}; + +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; + +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; + +use crate::xml as s3_xml; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// The multipart upload requested don't exists - #[error(display = "Upload not found")] + #[error("Upload not found")] NoSuchUpload, /// Precondition failed (e.g. x-amz-copy-source-if-match) - #[error(display = "At least one of the preconditions you specified did not hold")] + #[error("At least one of the preconditions you specified did not hold")] PreconditionFailed, /// Parts specified in CMU request do not match parts actually uploaded - #[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")] + #[error("Parts given to CompleteMultipartUpload do not match uploaded parts")] InvalidPart, /// Parts given to CompleteMultipartUpload were not in ascending order - #[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")] + #[error("Parts given to CompleteMultipartUpload were not in ascending order")] InvalidPartOrder, /// In CompleteMultipartUpload: not enough data /// (here we are more lenient than AWS S3) - #[error(display = "Proposed upload is smaller than the minimum allowed object size")] + #[error("Proposed upload is smaller than the minimum allowed object size")] EntityTooSmall, // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), /// The request used an invalid path - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8String(#[error(source)] std::string::FromUtf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8String(#[from] std::string::FromUtf8Error), /// The client sent invalid XML data - #[error(display = "Invalid XML: {}", _0)] + #[error("Invalid XML: {0}")] InvalidXml(String), - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + /// The client sent a range header with invalid value + #[error("Invalid HTTP range: {0:?}")] + InvalidRange((http_range::HttpRangeParseError, u64)), /// The client sent a range header with invalid value - #[error(display = "Invalid HTTP range: {:?}", _0)] - InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)), + #[error("Invalid encryption algorithm: {0:?}, should be AES256")] + InvalidEncryptionAlgorithm(String), + + /// The provided digest (checksum) value was invalid + #[error("Invalid digest: {0}")] + InvalidDigest(String), /// The client sent a request for an action not supported by garage - #[error(display = "Unimplemented action: {}", _0)] + #[error("Unimplemented action: {0}")] NotImplemented(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) +commonErrorDerivative!(Error); + +// Helper errors are always passed as internal errors by default. +// To pass the specific error code back to the client, use `pass_helper_error`. +impl From for Error { + fn from(err: HelperError) -> Error { + Error::Common(helper_error_as_internal(err)) } } -impl CommonErrorDerivative for Error {} - -impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), - e => Self::bad_request(format!("{}", e)), - } +impl From<(http_range::HttpRangeParseError, u64)> for Error { + fn from(err: (http_range::HttpRangeParseError, u64)) -> Error { + Error::InvalidRange(err) } } @@ -118,7 +125,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), - SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } @@ -143,9 +150,9 @@ impl Error { Error::NotImplemented(_) => "NotImplemented", Error::InvalidXml(_) => "MalformedXML", Error::InvalidRange(_) => "InvalidRange", - Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) | Error::InvalidHeader(_) => { - "InvalidRequest" - } + Error::InvalidDigest(_) => "InvalidDigest", + Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) => "InvalidRequest", + Error::InvalidEncryptionAlgorithm(_) => "InvalidEncryptionAlgorithmError", } } } @@ -163,10 +170,11 @@ impl ApiError for Error { | Error::InvalidPart | Error::InvalidPartOrder | Error::EntityTooSmall + | Error::InvalidDigest(_) + | Error::InvalidEncryptionAlgorithm(_) | Error::InvalidXml(_) | Error::InvalidUtf8Str(_) - | Error::InvalidUtf8String(_) - | Error::InvalidHeader(_) => StatusCode::BAD_REQUEST, + | Error::InvalidUtf8String(_) => StatusCode::BAD_REQUEST, } } @@ -189,22 +197,23 @@ impl ApiError for Error { } } - fn http_body(&self, garage_region: &str, path: &str) -> Body { + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody { let error = s3_xml::Error { code: s3_xml::Value(self.aws_code().to_string()), message: s3_xml::Value(format!("{}", self)), resource: Some(s3_xml::Value(path.to_string())), region: Some(s3_xml::Value(garage_region.to_string())), }; - Body::from(s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { + let error_str = s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { r#" - InternalError - XML encoding of error failed + InternalError + XML encoding of error failed - "# + "# .into() - })) + }); + error_body(error_str) } } diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 2a99551a..a1e4ce10 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -1,32 +1,57 @@ //! Function related to GET and HEAD requests +use std::collections::BTreeMap; +use std::convert::TryInto; use std::sync::Arc; -use std::time::{Duration, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use bytes::Bytes; use futures::future; -use futures::stream::{self, StreamExt}; +use futures::stream::{self, Stream, StreamExt}; use http::header::{ - ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, - IF_NONE_MATCH, LAST_MODIFIED, RANGE, + HeaderMap, HeaderName, ACCEPT_RANGES, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, + CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, EXPIRES, IF_MATCH, + IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE, LAST_MODIFIED, RANGE, }; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; use tokio::sync::mpsc; -use garage_rpc::rpc_helper::{netapp::stream::ByteStream, OrderTag}; +use garage_net::stream::ByteStream; +use garage_rpc::rpc_helper::OrderTag; use garage_table::EmptyKey; use garage_util::data::*; -use garage_util::error::OkOrMessage; +use garage_util::error::{Error as UtilError, OkOrMessage}; use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::s3::error::*; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; -const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; +use crate::api_server::ResBody; +use crate::copy::*; +use crate::encryption::EncryptionParams; +use crate::error::*; + +const X_AMZ_MP_PARTS_COUNT: HeaderName = HeaderName::from_static("x-amz-mp-parts-count"); + +#[derive(Default)] +pub struct GetObjectOverrides { + pub(crate) response_cache_control: Option, + pub(crate) response_content_disposition: Option, + pub(crate) response_content_encoding: Option, + pub(crate) response_content_language: Option, + pub(crate) response_content_type: Option, + pub(crate) response_expires: Option, +} fn object_headers( version: &ObjectVersion, version_meta: &ObjectVersionMeta, + meta_inner: &ObjectVersionMetaInner, + encryption: EncryptionParams, + checksum_mode: ChecksumMode, ) -> http::response::Builder { debug!("Version meta: {:?}", version_meta); @@ -34,7 +59,6 @@ fn object_headers( let date_str = httpdate::fmt_http_date(date); let mut resp = Response::builder() - .header(CONTENT_TYPE, version_meta.headers.content_type.to_string()) .header(LAST_MODIFIED, date_str) .header(ACCEPT_RANGES, "bytes".to_string()); @@ -42,60 +66,94 @@ fn object_headers( resp = resp.header(ETAG, format!("\"{}\"", version_meta.etag)); } - for (k, v) in version_meta.headers.other.iter() { - resp = resp.header(k, v.to_string()); + // When metadata is retrieved through the REST API, Amazon S3 combines headers that + // have the same name (ignoring case) into a comma-delimited list. + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html + let mut headers_by_name = BTreeMap::new(); + for (name, value) in meta_inner.headers.iter() { + let name_lower = name.to_ascii_lowercase(); + headers_by_name + .entry(name_lower) + .or_insert(vec![]) + .push(value.as_str()); } + for (name, values) in headers_by_name { + resp = resp.header(name, values.join(",")); + } + + if checksum_mode.enabled { + resp = add_checksum_response_headers(&meta_inner.checksum, resp); + } + + encryption.add_response_headers(&mut resp); + resp } -fn try_answer_cached( +/// Override headers according to specific query parameters, see +/// section "Overriding response header values through the request" in +/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html +fn getobject_override_headers( + overrides: GetObjectOverrides, + resp: &mut http::response::Builder, +) -> Result<(), Error> { + // TODO: this only applies for signed requests, so when we support + // anonymous access in the future we will have to do a permission check here + let overrides = [ + (CACHE_CONTROL, overrides.response_cache_control), + (CONTENT_DISPOSITION, overrides.response_content_disposition), + (CONTENT_ENCODING, overrides.response_content_encoding), + (CONTENT_LANGUAGE, overrides.response_content_language), + (CONTENT_TYPE, overrides.response_content_type), + (EXPIRES, overrides.response_expires), + ]; + for (hdr, val_opt) in overrides { + if let Some(val) = val_opt { + let val = val.try_into().ok_or_bad_request("invalid header value")?; + resp.headers_mut().unwrap().insert(hdr, val); + } + } + Ok(()) +} + +fn handle_http_precondition( version: &ObjectVersion, version_meta: &ObjectVersionMeta, - req: &Request, -) -> Option> { - // It is possible, and is even usually the case, [that both If-None-Match and - // If-Modified-Since] are present in a request. In this situation If-None-Match takes - // precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational - // being that etag based matching is more accurate, it has no issue with sub-second precision - // for instance (in case of very fast updates) - let cached = if let Some(none_match) = req.headers().get(IF_NONE_MATCH) { - let none_match = none_match.to_str().ok()?; - let expected = format!("\"{}\"", version_meta.etag); - let found = none_match - .split(',') - .map(str::trim) - .any(|etag| etag == expected || etag == "\"*\""); - found - } else if let Some(modified_since) = req.headers().get(IF_MODIFIED_SINCE) { - let modified_since = modified_since.to_str().ok()?; - let client_date = httpdate::parse_http_date(modified_since).ok()?; - let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp); - client_date >= server_date - } else { - false - }; + req: &Request<()>, +) -> Result>, Error> { + let precondition_headers = PreconditionHeaders::parse(req)?; - if cached { - Some( + if let Some(status_code) = precondition_headers.check(&version, &version_meta.etag)? { + Ok(Some( Response::builder() - .status(StatusCode::NOT_MODIFIED) - .body(Body::empty()) + .status(status_code) + .body(empty_body()) .unwrap(), - ) + )) } else { - None + Ok(None) } } /// Handle HEAD request pub async fn handle_head( + ctx: ReqCtx, + req: &Request<()>, + key: &str, + part_number: Option, +) -> Result, Error> { + handle_head_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number).await +} + +/// Handle HEAD request for website +pub async fn handle_head_without_ctx( garage: Arc, - req: &Request, + req: &Request<()>, bucket_id: Uuid, key: &str, part_number: Option, -) -> Result, Error> { +) -> Result, Error> { let object = garage .object_table .get(&bucket_id, &key.to_string()) @@ -120,25 +178,37 @@ pub async fn handle_head( _ => unreachable!(), }; - if let Some(cached) = try_answer_cached(object_version, version_meta, req) { - return Ok(cached); + if let Some(res) = handle_http_precondition(object_version, version_meta, req)? { + return Ok(res); } + let (encryption, headers) = + EncryptionParams::check_decrypt(&garage, req.headers(), &version_meta.encryption)?; + + let checksum_mode = checksum_mode(&req); + if let Some(pn) = part_number { match version_data { - ObjectVersionData::Inline(_, bytes) => { + ObjectVersionData::Inline(_, _) => { if pn != 1 { return Err(Error::InvalidPart); } - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", bytes.len())) - .header( - CONTENT_RANGE, - format!("bytes 0-{}/{}", bytes.len() - 1, bytes.len()), - ) - .header(X_AMZ_MP_PARTS_COUNT, "1") - .status(StatusCode::PARTIAL_CONTENT) - .body(Body::empty())?) + let bytes_len = version_meta.size; + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", bytes_len)) + .header( + CONTENT_RANGE, + format!("bytes 0-{}/{}", bytes_len - 1, bytes_len), + ) + .header(X_AMZ_MP_PARTS_COUNT, "1") + .status(StatusCode::PARTIAL_CONTENT) + .body(empty_body())?) } ObjectVersionData::FirstBlock(_, _) => { let version = garage @@ -146,44 +216,68 @@ pub async fn handle_head( .get(&object_version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; let (part_offset, part_end) = calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?; - let n_parts = version.parts_etags.items().len(); - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", part_end - part_offset)) - .header( - CONTENT_RANGE, - format!( - "bytes {}-{}/{}", - part_offset, - part_end - 1, - version_meta.size - ), - ) - .header(X_AMZ_MP_PARTS_COUNT, format!("{}", n_parts)) - .status(StatusCode::PARTIAL_CONTENT) - .body(Body::empty())?) + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", part_end - part_offset)) + .header( + CONTENT_RANGE, + format!( + "bytes {}-{}/{}", + part_offset, + part_end - 1, + version_meta.size + ), + ) + .header(X_AMZ_MP_PARTS_COUNT, format!("{}", version.n_parts()?)) + .status(StatusCode::PARTIAL_CONTENT) + .body(empty_body())?) } _ => unreachable!(), } } else { - Ok(object_headers(object_version, version_meta) - .header(CONTENT_LENGTH, format!("{}", version_meta.size)) - .status(StatusCode::OK) - .body(Body::empty())?) + Ok(object_headers( + object_version, + version_meta, + &headers, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", version_meta.size)) + .status(StatusCode::OK) + .body(empty_body())?) } } /// Handle GET request pub async fn handle_get( + ctx: ReqCtx, + req: &Request<()>, + key: &str, + part_number: Option, + overrides: GetObjectOverrides, +) -> Result, Error> { + handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await +} + +/// Handle GET request +pub async fn handle_get_without_ctx( garage: Arc, - req: &Request, + req: &Request<()>, bucket_id: Uuid, key: &str, part_number: Option, -) -> Result, Error> { + overrides: GetObjectOverrides, +) -> Result, Error> { let object = garage .object_table .get(&bucket_id, &key.to_string()) @@ -207,49 +301,136 @@ pub async fn handle_get( ObjectVersionData::FirstBlock(meta, _) => meta, }; - if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) { - return Ok(cached); + if let Some(res) = handle_http_precondition(last_v, last_v_meta, req)? { + return Ok(res); } + let (enc, headers) = + EncryptionParams::check_decrypt(&garage, req.headers(), &last_v_meta.encryption)?; + + let checksum_mode = checksum_mode(&req); + match (part_number, parse_range_header(req, last_v_meta.size)?) { - (Some(_), Some(_)) => { - return Err(Error::bad_request( - "Cannot specify both partNumber and Range header", - )); - } + (Some(_), Some(_)) => Err(Error::bad_request( + "Cannot specify both partNumber and Range header", + )), (Some(pn), None) => { - return handle_get_part(garage, last_v, last_v_data, last_v_meta, pn).await; - } - (None, Some(range)) => { - return handle_get_range( + handle_get_part( garage, last_v, last_v_data, last_v_meta, + enc, + &headers, + pn, + ChecksumMode { + // TODO: for multipart uploads, checksums of each part should be stored + // so that we can return the corresponding checksum here + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, + ) + .await + } + (None, Some(range)) => { + handle_get_range( + garage, + last_v, + last_v_data, + last_v_meta, + enc, + &headers, range.start, range.start + range.length, + ChecksumMode { + // TODO: for range queries that align with part boundaries, + // we should return the saved checksum of the part + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, ) - .await; + .await + } + (None, None) => { + handle_get_full( + garage, + last_v, + last_v_data, + last_v_meta, + enc, + &headers, + overrides, + checksum_mode, + ) + .await } - (None, None) => (), } +} - let resp_builder = object_headers(last_v, last_v_meta) - .header(CONTENT_LENGTH, format!("{}", last_v_meta.size)) - .status(StatusCode::OK); +pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> { + if version.deleted.get() { + // the version was deleted between when the object_table was consulted + // and now, this could mean the object was deleted, or overriden. + // Rather than say the key doesn't exist, return a transient error + // to signal the client to try again. + return Err(CommonError::InternalError(UtilError::Message( + "conflict/inconsistency between object and version state, version is deleted" + .to_string(), + )) + .into()); + } + Ok(()) +} - match &last_v_data { +async fn handle_get_full( + garage: Arc, + version: &ObjectVersion, + version_data: &ObjectVersionData, + version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, + overrides: GetObjectOverrides, + checksum_mode: ChecksumMode, +) -> Result, Error> { + let mut resp_builder = object_headers( + version, + version_meta, + &meta_inner, + encryption, + checksum_mode, + ) + .header(CONTENT_LENGTH, format!("{}", version_meta.size)) + .status(StatusCode::OK); + getobject_override_headers(overrides, &mut resp_builder)?; + + let stream = full_object_byte_stream(garage, version, version_data, encryption); + + Ok(resp_builder.body(response_body_from_stream(stream))?) +} + +pub fn full_object_byte_stream( + garage: Arc, + version: &ObjectVersion, + version_data: &ObjectVersionData, + encryption: EncryptionParams, +) -> ByteStream { + match &version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_, bytes) => { - let body: Body = Body::from(bytes.to_vec()); - Ok(resp_builder.body(body)?) + let bytes = bytes.to_vec(); + Box::pin(futures::stream::once(async move { + encryption + .decrypt_blob(&bytes) + .map(|x| Bytes::from(x.to_vec())) + .map_err(std_error_from_read_error) + })) } ObjectVersionData::FirstBlock(_, first_block_hash) => { - let (tx, rx) = mpsc::channel(2); + let (tx, rx) = mpsc::channel::(2); let order_stream = OrderTag::stream(); let first_block_hash = *first_block_hash; - let version_uuid = last_v.uuid; + let version_uuid = version.uuid; tokio::spawn(async move { match async { @@ -258,19 +439,19 @@ pub async fn handle_get( garage2.version_table.get(&version_uuid, &EmptyKey).await }); - let stream_block_0 = garage - .block_manager - .rpc_get_block_streaming(&first_block_hash, Some(order_stream.order(0))) + let stream_block_0 = encryption + .get_block(&garage, &first_block_hash, Some(order_stream.order(0))) .await?; + tx.send(stream_block_0) .await .ok_or_message("channel closed")?; let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) { - let stream_block_i = garage - .block_manager - .rpc_get_block_streaming(&vb.hash, Some(order_stream.order(i as u64))) + let stream_block_i = encryption + .get_block(&garage, &vb.hash, Some(order_stream.order(i as u64))) .await?; tx.send(stream_block_i) .await @@ -283,21 +464,20 @@ pub async fn handle_get( { Ok(()) => (), Err(e) => { - let err = std::io::Error::new( - std::io::ErrorKind::Other, - format!("Error while getting object data: {}", e), - ); - let _ = tx - .send(Box::pin(stream::once(future::ready(Err(err))))) - .await; + // TODO i think this is a bad idea, we should log + // an error and stop there. If the error happens to + // be exactly the size of what hasn't been streamed + // yet, the client will see the request as a + // success + // instead truncating the output notify the client + // something happened with their download, so that + // they can retry it + let _ = tx.send(error_stream_item(e)).await; } } }); - let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten(); - - let body = hyper::body::Body::wrap_stream(body_stream); - Ok(resp_builder.body(body)?) + Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx).flatten()) } } } @@ -307,10 +487,16 @@ async fn handle_get_range( version: &ObjectVersion, version_data: &ObjectVersionData, version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, begin: u64, end: u64, -) -> Result, Error> { - let resp_builder = object_headers(version, version_meta) + checksum_mode: ChecksumMode, +) -> Result, Error> { + // Here we do not use getobject_override_headers because we don't + // want to add any overridden headers (those should not be added + // when returning PARTIAL_CONTENT) + let resp_builder = object_headers(version, version_meta, meta_inner, encryption, checksum_mode) .header(CONTENT_LENGTH, format!("{}", end - begin)) .header( CONTENT_RANGE, @@ -321,8 +507,9 @@ async fn handle_get_range( match &version_data { ObjectVersionData::DeleteMarker => unreachable!(), ObjectVersionData::Inline(_meta, bytes) => { + let bytes = encryption.decrypt_blob(&bytes)?; if end as usize <= bytes.len() { - let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec()); + let body = bytes_body(bytes[begin as usize..end as usize].to_vec().into()); Ok(resp_builder.body(body)?) } else { Err(Error::internal_error( @@ -336,8 +523,9 @@ async fn handle_get_range( .get(&version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; - - let body = body_from_blocks_range(garage, version.blocks.items(), begin, end); + check_version_not_deleted(&version)?; + let body = + body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); Ok(resp_builder.body(body)?) } } @@ -348,16 +536,28 @@ async fn handle_get_part( object_version: &ObjectVersion, version_data: &ObjectVersionData, version_meta: &ObjectVersionMeta, + encryption: EncryptionParams, + meta_inner: &ObjectVersionMetaInner, part_number: u64, -) -> Result, Error> { - let resp_builder = - object_headers(object_version, version_meta).status(StatusCode::PARTIAL_CONTENT); + checksum_mode: ChecksumMode, +) -> Result, Error> { + // Same as for get_range, no getobject_override_headers + let resp_builder = object_headers( + object_version, + version_meta, + meta_inner, + encryption, + checksum_mode, + ) + .status(StatusCode::PARTIAL_CONTENT); match version_data { ObjectVersionData::Inline(_, bytes) => { if part_number != 1 { return Err(Error::InvalidPart); } + let bytes = encryption.decrypt_blob(&bytes)?; + assert_eq!(bytes.len() as u64, version_meta.size); Ok(resp_builder .header(CONTENT_LENGTH, format!("{}", bytes.len())) .header( @@ -365,7 +565,7 @@ async fn handle_get_part( format!("bytes {}-{}/{}", 0, bytes.len() - 1, bytes.len()), ) .header(X_AMZ_MP_PARTS_COUNT, "1") - .body(Body::from(bytes.to_vec()))?) + .body(bytes_body(bytes.into_owned().into()))?) } ObjectVersionData::FirstBlock(_, _) => { let version = garage @@ -374,11 +574,13 @@ async fn handle_get_part( .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; + let (begin, end) = calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?; - let n_parts = version.parts_etags.items().len(); - let body = body_from_blocks_range(garage, version.blocks.items(), begin, end); + let body = + body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); Ok(resp_builder .header(CONTENT_LENGTH, format!("{}", end - begin)) @@ -386,7 +588,7 @@ async fn handle_get_part( CONTENT_RANGE, format!("bytes {}-{}/{}", begin, end - 1, version_meta.size), ) - .header(X_AMZ_MP_PARTS_COUNT, format!("{}", n_parts)) + .header(X_AMZ_MP_PARTS_COUNT, format!("{}", version.n_parts()?)) .body(body)?) } _ => unreachable!(), @@ -394,7 +596,7 @@ async fn handle_get_part( } fn parse_range_header( - req: &Request, + req: &Request<()>, total_size: u64, ) -> Result, Error> { let range = match req.headers().get(RANGE) { @@ -431,19 +633,34 @@ fn calculate_part_bounds(v: &Version, part_number: u64) -> Option<(u64, u64)> { None } +struct ChecksumMode { + enabled: bool, +} + +fn checksum_mode(req: &Request<()>) -> ChecksumMode { + ChecksumMode { + enabled: req + .headers() + .get(X_AMZ_CHECKSUM_MODE) + .map(|x| x == "ENABLED") + .unwrap_or(false), + } +} + fn body_from_blocks_range( garage: Arc, + encryption: EncryptionParams, all_blocks: &[(VersionBlockKey, VersionBlock)], begin: u64, end: u64, -) -> Body { +) -> ResBody { // We will store here the list of blocks that have an intersection with the requested // range, as well as their "true offset", which is their actual offset in the complete // file (whereas block.offset designates the offset of the block WITHIN THE PART // block.part_number, which is not the same in the case of a multipart upload) let mut blocks: Vec<(VersionBlock, u64)> = Vec::with_capacity(std::cmp::min( all_blocks.len(), - 4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size as u64, 1024)) as usize, + 4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size, 1024)) as usize, )); let mut block_offset: u64 = 0; for (_, b) in all_blocks.iter() { @@ -454,21 +671,20 @@ fn body_from_blocks_range( if block_offset < end && block_offset + b.size > begin { blocks.push((*b, block_offset)); } - block_offset += b.size as u64; + block_offset += b.size; } let order_stream = OrderTag::stream(); - let body_stream = futures::stream::iter(blocks) - .enumerate() - .map(move |(i, (block, block_offset))| { - let garage = garage.clone(); - async move { - garage - .block_manager - .rpc_get_block_streaming(&block.hash, Some(order_stream.order(i as u64))) - .await - .unwrap_or_else(|e| error_stream(i, e)) - .scan(block_offset, move |chunk_offset, chunk| { + let (tx, rx) = mpsc::channel::(2); + + tokio::spawn(async move { + match async { + for (i, (block, block_offset)) in blocks.iter().enumerate() { + let block_stream = encryption + .get_block(&garage, &block.hash, Some(order_stream.order(i as u64))) + .await?; + let block_stream = block_stream + .scan(*block_offset, move |chunk_offset, chunk| { let r = match chunk { Ok(chunk_bytes) => { let chunk_len = chunk_bytes.len() as u64; @@ -504,20 +720,168 @@ fn body_from_blocks_range( }; futures::future::ready(r) }) - .filter_map(futures::future::ready) + .filter_map(futures::future::ready); + + let block_stream: ByteStream = Box::pin(block_stream); + tx.send(Box::pin(block_stream)) + .await + .ok_or_message("channel closed")?; } + + Ok::<(), Error>(()) + } + .await + { + Ok(()) => (), + Err(e) => { + let _ = tx.send(error_stream_item(e)).await; + } + } + }); + + response_body_from_block_stream(rx) +} + +fn response_body_from_block_stream(rx: mpsc::Receiver) -> ResBody { + let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten(); + response_body_from_stream(body_stream) +} + +fn response_body_from_stream(stream: S) -> ResBody +where + S: Stream> + Send + Sync + 'static, +{ + let body_stream = stream.map(|x| { + x.map(hyper::body::Frame::data) + .map_err(|e| Error::from(garage_util::error::Error::from(e))) + }); + ResBody::new(http_body_util::StreamBody::new(body_stream)) +} + +fn error_stream_item(e: E) -> ByteStream { + Box::pin(stream::once(future::ready(Err(std_error_from_read_error( + e, + ))))) +} + +fn std_error_from_read_error(e: E) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Error while reading object data: {}", e), + ) +} + +// ---- + +pub struct PreconditionHeaders { + if_match: Option>, + if_modified_since: Option, + if_none_match: Option>, + if_unmodified_since: Option, +} + +impl PreconditionHeaders { + fn parse(req: &Request) -> Result { + Self::parse_with( + req.headers(), + &IF_MATCH, + &IF_NONE_MATCH, + &IF_MODIFIED_SINCE, + &IF_UNMODIFIED_SINCE, + ) + } + + pub(crate) fn parse_copy_source(req: &Request) -> Result { + Self::parse_with( + req.headers(), + &X_AMZ_COPY_SOURCE_IF_MATCH, + &X_AMZ_COPY_SOURCE_IF_NONE_MATCH, + &X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE, + &X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE, + ) + } + + fn parse_with( + headers: &HeaderMap, + hdr_if_match: &HeaderName, + hdr_if_none_match: &HeaderName, + hdr_if_modified_since: &HeaderName, + hdr_if_unmodified_since: &HeaderName, + ) -> Result { + Ok(Self { + if_match: headers + .get(hdr_if_match) + .map(|x| x.to_str()) + .transpose()? + .map(|x| { + x.split(',') + .map(|m| m.trim().trim_matches('"').to_string()) + .collect::>() + }), + if_none_match: headers + .get(hdr_if_none_match) + .map(|x| x.to_str()) + .transpose()? + .map(|x| { + x.split(',') + .map(|m| m.trim().trim_matches('"').to_string()) + .collect::>() + }), + if_modified_since: headers + .get(hdr_if_modified_since) + .map(|x| x.to_str()) + .transpose()? + .map(httpdate::parse_http_date) + .transpose() + .ok_or_bad_request("Invalid date in if-modified-since")?, + if_unmodified_since: headers + .get(hdr_if_unmodified_since) + .map(|x| x.to_str()) + .transpose()? + .map(httpdate::parse_http_date) + .transpose() + .ok_or_bad_request("Invalid date in if-unmodified-since")?, }) - .buffered(2) - .flatten(); + } - hyper::body::Body::wrap_stream(body_stream) -} + fn check(&self, v: &ObjectVersion, etag: &str) -> Result, Error> { + // we store date with ms precision, but headers are precise to the second: truncate + // the timestamp to handle the same-second edge case + let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000); -fn error_stream(i: usize, e: garage_util::error::Error) -> ByteStream { - Box::pin(futures::stream::once(async move { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("Could not get block {}: {}", i, e), - )) - })) + // Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6 + + if let Some(im) = &self.if_match { + // Step 1: if-match is present + if !im.iter().any(|x| x == etag || x == "*") { + return Ok(Some(StatusCode::PRECONDITION_FAILED)); + } + } else if let Some(ius) = &self.if_unmodified_since { + // Step 2: if-unmodified-since is present, and if-match is absent + if v_date > *ius { + return Ok(Some(StatusCode::PRECONDITION_FAILED)); + } + } + + if let Some(inm) = &self.if_none_match { + // Step 3: if-none-match is present + if inm.iter().any(|x| x == etag || x == "*") { + return Ok(Some(StatusCode::NOT_MODIFIED)); + } + } else if let Some(ims) = &self.if_modified_since { + // Step 4: if-modified-since is present, and if-none-match is absent + if v_date <= *ims { + return Ok(Some(StatusCode::NOT_MODIFIED)); + } + } + + Ok(None) + } + + pub(crate) fn check_copy_source(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> { + match self.check(v, etag)? { + Some(_) => Err(Error::PreconditionFailed), + None => Ok(()), + } + } } diff --git a/src/api/s3/mod.rs b/src/api/s3/lib.rs similarity index 61% rename from src/api/s3/mod.rs rename to src/api/s3/lib.rs index 7b56d4d8..83f684f8 100644 --- a/src/api/s3/mod.rs +++ b/src/api/s3/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate tracing; + pub mod api_server; pub mod error; @@ -6,10 +9,13 @@ mod copy; pub mod cors; mod delete; pub mod get; +mod lifecycle; mod list; +mod multipart; mod post_object; mod put; -mod website; +pub mod website; +mod encryption; mod router; pub mod xml; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs new file mode 100644 index 00000000..ccda6cfd --- /dev/null +++ b/src/api/s3/lifecycle.rs @@ -0,0 +1,400 @@ +use quick_xml::de::from_reader; + +use hyper::{Request, Response, StatusCode}; + +use serde::{Deserialize, Serialize}; + +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; + +use garage_model::bucket_table::{ + parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, + LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule, +}; + +pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + + if let Some(lifecycle) = bucket_params.lifecycle_config.get() { + let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle); + let xml = to_xml_with_header(&wc)?; + Ok(Response::builder() + .status(StatusCode::OK) + .header(http::header::CONTENT_TYPE, "application/xml") + .body(string_body(xml))?) + } else { + Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body(empty_body())?) + } +} + +pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.lifecycle_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; + + Ok(Response::builder() + .status(StatusCode::NO_CONTENT) + .body(empty_body())?) +} + +pub async fn handle_put_lifecycle( + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + + let body = req.into_body().collect().await?; + + let conf: LifecycleConfiguration = from_reader(&body as &[u8])?; + let config = conf + .validate_into_garage_lifecycle_config() + .ok_or_bad_request("Invalid lifecycle configuration")?; + + bucket_params.lifecycle_config.update(Some(config)); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; + + Ok(Response::builder() + .status(StatusCode::OK) + .body(empty_body())?) +} + +// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct LifecycleConfiguration { + #[serde(serialize_with = "xmlns_tag", skip_deserializing)] + pub xmlns: (), + #[serde(rename = "Rule")] + pub lifecycle_rules: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct LifecycleRule { + #[serde(rename = "ID")] + pub id: Option, + #[serde(rename = "Status")] + pub status: Value, + #[serde(rename = "Filter", default)] + pub filter: Option, + #[serde(rename = "Expiration", default)] + pub expiration: Option, + #[serde(rename = "AbortIncompleteMultipartUpload", default)] + pub abort_incomplete_mpu: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct Filter { + #[serde(rename = "And")] + pub and: Option>, + #[serde(rename = "Prefix")] + pub prefix: Option, + #[serde(rename = "ObjectSizeGreaterThan")] + pub size_gt: Option, + #[serde(rename = "ObjectSizeLessThan")] + pub size_lt: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Expiration { + #[serde(rename = "Days")] + pub days: Option, + #[serde(rename = "Date")] + pub at_date: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct AbortIncompleteMpu { + #[serde(rename = "DaysAfterInitiation")] + pub days: IntValue, +} + +impl LifecycleConfiguration { + pub fn validate_into_garage_lifecycle_config( + self, + ) -> Result, &'static str> { + let mut ret = vec![]; + for rule in self.lifecycle_rules { + ret.push(rule.validate_into_garage_lifecycle_rule()?); + } + Ok(ret) + } + + pub fn from_garage_lifecycle_config(config: &[GarageLifecycleRule]) -> Self { + Self { + xmlns: (), + lifecycle_rules: config + .iter() + .map(LifecycleRule::from_garage_lifecycle_rule) + .collect(), + } + } +} + +impl LifecycleRule { + pub fn validate_into_garage_lifecycle_rule(self) -> Result { + let enabled = match self.status.0.as_str() { + "Enabled" => true, + "Disabled" => false, + _ => return Err("invalid value for "), + }; + + let filter = self + .filter + .map(Filter::validate_into_garage_lifecycle_filter) + .transpose()? + .unwrap_or_default(); + + let abort_incomplete_mpu_days = self.abort_incomplete_mpu.map(|x| x.days.0 as usize); + + let expiration = self + .expiration + .map(Expiration::validate_into_garage_lifecycle_expiration) + .transpose()?; + + Ok(GarageLifecycleRule { + id: self.id.map(|x| x.0), + enabled, + filter, + abort_incomplete_mpu_days, + expiration, + }) + } + + pub fn from_garage_lifecycle_rule(rule: &GarageLifecycleRule) -> Self { + Self { + id: rule.id.as_deref().map(Value::from), + status: if rule.enabled { + Value::from("Enabled") + } else { + Value::from("Disabled") + }, + filter: Filter::from_garage_lifecycle_filter(&rule.filter), + abort_incomplete_mpu: rule + .abort_incomplete_mpu_days + .map(|days| AbortIncompleteMpu { + days: IntValue(days as i64), + }), + expiration: rule + .expiration + .as_ref() + .map(Expiration::from_garage_lifecycle_expiration), + } + } +} + +impl Filter { + pub fn count(&self) -> i32 { + fn count(x: &Option) -> i32 { + x.as_ref().map(|_| 1).unwrap_or(0) + } + count(&self.prefix) + count(&self.size_gt) + count(&self.size_lt) + } + + pub fn validate_into_garage_lifecycle_filter( + self, + ) -> Result { + if self.count() > 0 && self.and.is_some() { + Err("Filter tag cannot contain both and another condition") + } else if let Some(and) = self.and { + if and.and.is_some() { + return Err("Nested tags"); + } + Ok(and.internal_into_garage_lifecycle_filter()) + } else if self.count() > 1 { + Err("Multiple Filter conditions must be wrapped in an tag") + } else { + Ok(self.internal_into_garage_lifecycle_filter()) + } + } + + fn internal_into_garage_lifecycle_filter(self) -> GarageLifecycleFilter { + GarageLifecycleFilter { + prefix: self.prefix.map(|x| x.0), + size_gt: self.size_gt.map(|x| x.0 as u64), + size_lt: self.size_lt.map(|x| x.0 as u64), + } + } + + pub fn from_garage_lifecycle_filter(rule: &GarageLifecycleFilter) -> Option { + let filter = Filter { + and: None, + prefix: rule.prefix.as_deref().map(Value::from), + size_gt: rule.size_gt.map(|x| IntValue(x as i64)), + size_lt: rule.size_lt.map(|x| IntValue(x as i64)), + }; + match filter.count() { + 0 => None, + 1 => Some(filter), + _ => Some(Filter { + and: Some(Box::new(filter)), + ..Default::default() + }), + } + } +} + +impl Expiration { + pub fn validate_into_garage_lifecycle_expiration( + self, + ) -> Result { + match (self.days, self.at_date) { + (Some(_), Some(_)) => Err("cannot have both and in "), + (None, None) => Err(" must contain either or "), + (Some(days), None) => Ok(GarageLifecycleExpiration::AfterDays(days.0 as usize)), + (None, Some(date)) => { + parse_lifecycle_date(&date.0)?; + Ok(GarageLifecycleExpiration::AtDate(date.0)) + } + } + } + + pub fn from_garage_lifecycle_expiration(exp: &GarageLifecycleExpiration) -> Self { + match exp { + GarageLifecycleExpiration::AfterDays(days) => Expiration { + days: Some(IntValue(*days as i64)), + at_date: None, + }, + GarageLifecycleExpiration::AtDate(date) => Expiration { + days: None, + at_date: Some(Value(date.to_string())), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use quick_xml::de::from_str; + + #[test] + fn test_deserialize_lifecycle_config() -> Result<(), Error> { + let message = r#" + + + id1 + Enabled + + documents/ + + + 7 + + + + id2 + Enabled + + + logs/ + 1000000 + + + + 365 + + +"#; + let conf: LifecycleConfiguration = from_str(message).unwrap(); + let ref_value = LifecycleConfiguration { + xmlns: (), + lifecycle_rules: vec![ + LifecycleRule { + id: Some("id1".into()), + status: "Enabled".into(), + filter: Some(Filter { + prefix: Some("documents/".into()), + ..Default::default() + }), + expiration: None, + abort_incomplete_mpu: Some(AbortIncompleteMpu { days: IntValue(7) }), + }, + LifecycleRule { + id: Some("id2".into()), + status: "Enabled".into(), + filter: Some(Filter { + and: Some(Box::new(Filter { + prefix: Some("logs/".into()), + size_gt: Some(IntValue(1000000)), + ..Default::default() + })), + ..Default::default() + }), + expiration: Some(Expiration { + days: Some(IntValue(365)), + at_date: None, + }), + abort_incomplete_mpu: None, + }, + ], + }; + assert_eq! { + ref_value, + conf + }; + + let message2 = to_xml_with_header(&ref_value)?; + + let cleanup = |c: &str| c.replace(char::is_whitespace, ""); + assert_eq!(cleanup(message), cleanup(&message2)); + + // Check validation + let validated = ref_value + .validate_into_garage_lifecycle_config() + .ok_or_bad_request("invalid xml config")?; + + let ref_config = vec![ + GarageLifecycleRule { + id: Some("id1".into()), + enabled: true, + filter: GarageLifecycleFilter { + prefix: Some("documents/".into()), + ..Default::default() + }, + expiration: None, + abort_incomplete_mpu_days: Some(7), + }, + GarageLifecycleRule { + id: Some("id2".into()), + enabled: true, + filter: GarageLifecycleFilter { + prefix: Some("logs/".into()), + size_gt: Some(1000000), + ..Default::default() + }, + expiration: Some(GarageLifecycleExpiration::AfterDays(365)), + abort_incomplete_mpu_days: None, + }, + ]; + assert_eq!(validated, ref_config); + + let message3 = to_xml_with_header(&LifecycleConfiguration::from_garage_lifecycle_config( + &validated, + ))?; + assert_eq!(cleanup(message), cleanup(&message3)); + + Ok(()) + } +} diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index 5cb0d65a..94c2c895 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -1,26 +1,26 @@ -use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet}; use std::iter::{Iterator, Peekable}; -use std::sync::Arc; use base64::prelude::*; -use hyper::{Body, Response}; +use hyper::{Request, Response}; use garage_util::data::*; use garage_util::error::Error as GarageError; use garage_util::time::*; -use garage_model::garage::Garage; +use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; -use garage_model::s3::version_table::Version; -use garage_table::{EmptyKey, EnumerationOrder}; +use garage_table::EnumerationOrder; -use crate::encoding::*; -use crate::helpers::key_after_prefix; -use crate::s3::error::*; -use crate::s3::put as s3_put; -use crate::s3::xml as s3_xml; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::multipart as s3_multipart; +use crate::xml as s3_xml; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; @@ -54,7 +54,6 @@ pub struct ListMultipartUploadsQuery { #[derive(Debug)] pub struct ListPartsQuery { pub bucket_name: String, - pub bucket_id: Uuid, pub key: String, pub upload_id: String, pub part_number_marker: Option, @@ -62,9 +61,10 @@ pub struct ListPartsQuery { } pub async fn handle_list( - garage: Arc, + ctx: ReqCtx, query: &ListObjectsQuery, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; let io = |bucket, key, count| { let t = &garage.object_table; async move { @@ -163,20 +163,24 @@ pub async fn handle_list( let xml = s3_xml::to_xml_with_header(&result)?; Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } pub async fn handle_list_multipart_upload( - garage: Arc, + ctx: ReqCtx, query: &ListMultipartUploadsQuery, -) -> Result, Error> { +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + let io = |bucket, key, count| { let t = &garage.object_table; async move { t.get_range( &bucket, key, - Some(ObjectFilter::IsUploading), + Some(ObjectFilter::IsUploading { + check_multipart: Some(true), + }), count, EnumerationOrder::Forward, ) @@ -263,42 +267,82 @@ pub async fn handle_list_multipart_upload( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } pub async fn handle_list_parts( - garage: Arc, + ctx: ReqCtx, + req: Request, query: &ListPartsQuery, -) -> Result, Error> { +) -> Result, Error> { debug!("ListParts {:?}", query); - let upload_id = s3_put::decode_upload_id(&query.upload_id)?; + let upload_id = s3_multipart::decode_upload_id(&query.upload_id)?; - let (object, version) = futures::try_join!( - garage.object_table.get(&query.bucket_id, &query.key), - garage.version_table.get(&upload_id, &EmptyKey), - )?; + let (_, object_version, mpu) = s3_multipart::get_upload(&ctx, &query.key, &upload_id).await?; - let (info, next) = fetch_part_info(query, object, version, upload_id)?; + let object_encryption = match object_version.state { + ObjectVersionState::Uploading { encryption, .. } => encryption, + _ => unreachable!(), + }; + let encryption_res = + EncryptionParams::check_decrypt(&ctx.garage, req.headers(), &object_encryption); + + let (info, next) = fetch_part_info(query, &mpu)?; let result = s3_xml::ListPartsResult { xmlns: (), + + // Query parameters bucket: s3_xml::Value(query.bucket_name.to_string()), key: s3_xml::Value(query.key.to_string()), upload_id: s3_xml::Value(query.upload_id.to_string()), part_number_marker: query.part_number_marker.map(|e| s3_xml::IntValue(e as i64)), - next_part_number_marker: next.map(|e| s3_xml::IntValue(e as i64)), max_parts: s3_xml::IntValue(query.max_parts as i64), - is_truncated: s3_xml::Value(next.map(|_| "true").unwrap_or("false").to_string()), + + // Result values + next_part_number_marker: next.map(|e| s3_xml::IntValue(e as i64)), + is_truncated: s3_xml::Value(format!("{}", next.is_some())), parts: info .iter() - .map(|part| s3_xml::PartItem { - etag: s3_xml::Value(format!("\"{}\"", part.etag)), - last_modified: s3_xml::Value(msec_to_rfc3339(part.timestamp)), - part_number: s3_xml::IntValue(part.part_number as i64), - size: s3_xml::IntValue(part.size as i64), + .map(|part| { + // hide checksum if object is encrypted and the decryption + // keys are not provided + let checksum = part.checksum.filter(|_| encryption_res.is_ok()); + s3_xml::PartItem { + etag: s3_xml::Value(format!("\"{}\"", part.etag)), + last_modified: s3_xml::Value(msec_to_rfc3339(part.timestamp)), + part_number: s3_xml::IntValue(part.part_number as i64), + size: s3_xml::IntValue(part.size as i64), + checksum_crc32: match &checksum { + Some(ChecksumValue::Crc32(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_crc32c: match &checksum { + Some(ChecksumValue::Crc32c(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_sha1: match &checksum { + Some(ChecksumValue::Sha1(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + checksum_sha256: match &checksum { + Some(ChecksumValue::Sha256(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, + } }) .collect(), + + // Dummy result values (unsupported features) initiator: s3_xml::Initiator { display_name: s3_xml::Value(DUMMY_NAME.to_string()), id: s3_xml::Value(DUMMY_KEY.to_string()), @@ -314,7 +358,7 @@ pub async fn handle_list_parts( Ok(Response::builder() .header("Content-Type", "application/xml") - .body(Body::from(xml.into_bytes()))?) + .body(string_body(xml))?) } /* @@ -335,11 +379,12 @@ struct UploadInfo { } #[derive(Debug, PartialEq)] -struct PartInfo { - etag: String, +struct PartInfo<'a> { + etag: &'a str, timestamp: u64, part_number: u64, size: u64, + checksum: Option, } enum ExtractionResult { @@ -353,7 +398,7 @@ enum ExtractionResult { key: String, }, // Fallback key is used for legacy APIs that only support - // exlusive pagination (and not inclusive one). + // exclusive pagination (and not inclusive one). SkipTo { key: String, fallback_key: Option, @@ -363,7 +408,7 @@ enum ExtractionResult { #[derive(PartialEq, Clone, Debug)] enum RangeBegin { // Fallback key is used for legacy APIs that only support - // exlusive pagination (and not inclusive one). + // exclusive pagination (and not inclusive one). IncludingKey { key: String, fallback_key: Option, @@ -421,8 +466,10 @@ where // Drop the first key if needed // Only AfterKey requires it according to the S3 spec and our implem. match (&cursor, iter.peek()) { - (RangeBegin::AfterKey { key }, Some(object)) if &object.key == key => iter.next(), - (_, _) => None, + (RangeBegin::AfterKey { key }, Some(object)) if &object.key == key => { + iter.next(); + } + _ => (), }; while let Some(object) = iter.peek() { @@ -431,16 +478,22 @@ where return Ok(None); } - cursor = match acc.extract(query, &cursor, &mut iter) { - ExtractionResult::Extracted { key } => RangeBegin::AfterKey { key }, + match acc.extract(query, &cursor, &mut iter) { + ExtractionResult::Extracted { key } => { + cursor = RangeBegin::AfterKey { key }; + } ExtractionResult::SkipTo { key, fallback_key } => { - RangeBegin::IncludingKey { key, fallback_key } + cursor = RangeBegin::IncludingKey { key, fallback_key }; } ExtractionResult::FilledAtUpload { key, upload } => { - return Ok(Some(RangeBegin::AfterUpload { key, upload })) + return Ok(Some(RangeBegin::AfterUpload { key, upload })); + } + ExtractionResult::Filled => { + return Ok(Some(cursor)); + } + ExtractionResult::NoMore => { + return Ok(None); } - ExtractionResult::Filled => return Ok(Some(cursor)), - ExtractionResult::NoMore => return Ok(None), }; } @@ -456,107 +509,52 @@ where } } -fn fetch_part_info( +fn fetch_part_info<'a>( query: &ListPartsQuery, - object: Option, - version: Option, - upload_id: Uuid, -) -> Result<(Vec, Option), Error> { - // Check results - let object = object.ok_or(Error::NoSuchKey)?; + mpu: &'a MultipartUpload, +) -> Result<(Vec>, Option), Error> { + assert!((1..=1000).contains(&query.max_parts)); // see s3/api_server.rs - let obj_version = object - .versions() - .iter() - .find(|v| v.uuid == upload_id && v.is_uploading()) - .ok_or(Error::NoSuchUpload)?; - - let version = version.ok_or(Error::NoSuchKey)?; - - // Cut the beginning of our 2 vectors if required - let (etags, blocks) = match &query.part_number_marker { - Some(marker) => { - let next = marker + 1; - - let part_idx = into_ok_or_err( - version - .parts_etags - .items() - .binary_search_by(|(part_num, _)| part_num.cmp(&next)), - ); - let parts = &version.parts_etags.items()[part_idx..]; - - let block_idx = into_ok_or_err( - version - .blocks - .items() - .binary_search_by(|(vkey, _)| vkey.part_number.cmp(&next)), - ); - let blocks = &version.blocks.items()[block_idx..]; - - (parts, blocks) - } - None => (version.parts_etags.items(), version.blocks.items()), - }; - - // Use the block vector to compute a (part_number, size) vector - let mut size = Vec::<(u64, u64)>::new(); - blocks.iter().for_each(|(key, val)| { - let mut new_size = val.size; - match size.pop() { - Some((part_number, size)) if part_number == key.part_number => new_size += size, - Some(v) => size.push(v), - None => (), - } - size.push((key.part_number, new_size)) - }); - - // Merge the etag vector and size vector to build a PartInfo vector - let max_parts = query.max_parts as usize; - let (mut etag_iter, mut size_iter) = (etags.iter().peekable(), size.iter().peekable()); - - let mut info = Vec::::with_capacity(max_parts); - - while info.len() < max_parts { - match (etag_iter.peek(), size_iter.peek()) { - (Some((ep, etag)), Some((sp, size))) => match ep.cmp(sp) { - Ordering::Less => { - debug!("ETag information ignored due to missing corresponding block information. Query: {:?}", query); - etag_iter.next(); + // Parse multipart upload part list, removing parts not yet finished + // and failed part uploads that were overwritten + let mut parts: Vec> = Vec::with_capacity(mpu.parts.items().len()); + for (pk, p) in mpu.parts.items().iter() { + if let (Some(etag), Some(size)) = (&p.etag, p.size) { + let part_info = PartInfo { + part_number: pk.part_number, + timestamp: pk.timestamp, + etag, + size, + checksum: p.checksum, + }; + match parts.last_mut() { + Some(lastpart) if lastpart.part_number == pk.part_number => { + *lastpart = part_info; } - Ordering::Equal => { - info.push(PartInfo { - etag: etag.to_string(), - timestamp: obj_version.timestamp, - part_number: *ep, - size: *size, - }); - etag_iter.next(); - size_iter.next(); + _ => { + parts.push(part_info); } - Ordering::Greater => { - debug!("Block information ignored due to missing corresponding ETag information. Query: {:?}", query); - size_iter.next(); - } - }, - (None, None) => return Ok((info, None)), - _ => { - debug!( - "Additional block or ETag information ignored. Query: {:?}", - query - ); - return Ok((info, None)); } } } - match info.last() { - Some(part_info) => { - let pagination = Some(part_info.part_number); - Ok((info, pagination)) - } - None => Ok((info, None)), + // Cut the beginning if we have a marker + if let Some(marker) = &query.part_number_marker { + let next = marker + 1; + let part_idx = parts + .binary_search_by(|part| part.part_number.cmp(&next)) + .unwrap_or_else(|x| x); + parts = parts.split_off(part_idx); } + + // Cut the end if we have too many parts + if parts.len() > query.max_parts as usize { + parts.truncate(query.max_parts as usize); + let pagination = Some(parts.last().unwrap().part_number); + return Ok((parts, pagination)); + } + + Ok((parts, None)) } /* @@ -570,8 +568,8 @@ fn fetch_part_info( /// This key can be the prefix in the base case, or intermediate /// points in the dataset if we are continuing a previous listing. impl ListObjectsQuery { - fn build_accumulator(&self) -> Accumulator { - Accumulator::::new(self.common.page_size) + fn build_accumulator(&self) -> ObjectAccumulator { + ObjectAccumulator::new(self.common.page_size) } fn begin(&self) -> Result { @@ -580,9 +578,10 @@ impl ListObjectsQuery { // In V2 mode, the continuation token is defined as an opaque // string in the spec, so we can do whatever we want with it. // In our case, it is defined as either [ or ] (for include + // or exclude), followed by a base64-encoded string // representing the key to start with. - (Some(token), _) => match &token[..1] { - "[" => Ok(RangeBegin::IncludingKey { + (Some(token), _) => match &token.get(..1) { + Some("[") => Ok(RangeBegin::IncludingKey { key: String::from_utf8( BASE64_STANDARD .decode(token[1..].as_bytes()) @@ -590,7 +589,7 @@ impl ListObjectsQuery { )?, fallback_key: None, }), - "]" => Ok(RangeBegin::AfterKey { + Some("]") => Ok(RangeBegin::AfterKey { key: String::from_utf8( BASE64_STANDARD .decode(token[1..].as_bytes()) @@ -631,8 +630,8 @@ impl ListObjectsQuery { } impl ListMultipartUploadsQuery { - fn build_accumulator(&self) -> Accumulator { - Accumulator::::new(self.common.page_size) + fn build_accumulator(&self) -> UploadAccumulator { + UploadAccumulator::new(self.common.page_size) } fn begin(&self) -> Result { @@ -651,7 +650,7 @@ impl ListMultipartUploadsQuery { }), uuid => Ok(RangeBegin::AfterUpload { key: key_marker.to_string(), - upload: s3_put::decode_upload_id(uuid)?, + upload: s3_multipart::decode_upload_id(uuid)?, }), }, @@ -716,6 +715,7 @@ impl Accumulator { Some(p) => p, None => return None, }; + assert!(pfx.starts_with(&query.prefix)); // Try to register this prefix // If not possible, we can return early @@ -726,8 +726,11 @@ impl Accumulator { // We consume the whole common prefix from the iterator let mut last_pfx_key = &object.key; loop { - last_pfx_key = match objects.peek() { - Some(o) if o.key.starts_with(pfx) => &o.key, + match objects.peek() { + Some(o) if o.key.starts_with(pfx) => { + last_pfx_key = &o.key; + objects.next(); + } Some(_) => { return Some(ExtractionResult::Extracted { key: last_pfx_key.to_owned(), @@ -743,8 +746,6 @@ impl Accumulator { } } }; - - objects.next(); } } @@ -759,12 +760,11 @@ impl Accumulator { } // Otherwise, we need to check if we can add it - match self.is_full() { - true => false, - false => { - self.common_prefixes.insert(key); - true - } + if self.is_full() { + false + } else { + self.common_prefixes.insert(key); + true } } @@ -772,12 +772,11 @@ impl Accumulator { // It is impossible to add twice a key, this is an error assert!(!self.keys.contains_key(&key)); - match self.is_full() { - true => false, - false => { - self.keys.insert(key, value); - true - } + if self.is_full() { + false + } else { + self.keys.insert(key, value); + true } } } @@ -794,6 +793,7 @@ impl ExtractAccumulator for ObjectAccumulator { } let object = objects.next().expect("This iterator can not be empty as it is checked earlier in the code. This is a logic bug, please report it."); + assert!(object.key.starts_with(&query.prefix)); let version = match object.versions().iter().find(|x| x.is_data()) { Some(v) => v, @@ -843,7 +843,7 @@ impl ExtractAccumulator for UploadAccumulator { let mut uploads_for_key = object .versions() .iter() - .filter(|x| x.is_uploading()) + .filter(|x| x.is_uploading(Some(true))) .collect::>(); // S3 logic requires lexicographically sorted upload ids. @@ -918,14 +918,6 @@ impl ExtractAccumulator for UploadAccumulator { * Utility functions */ -/// This is a stub for Result::into_ok_or_err that is not yet in Rust stable -fn into_ok_or_err(r: Result) -> T { - match r { - Ok(r) => r, - Err(r) => r, - } -} - /// Returns the common prefix of the object given the query prefix and delimiter fn common_prefix<'a>(object: &'a Object, query: &ListQueryCommon) -> Option<&'a str> { match &query.delimiter { @@ -951,7 +943,6 @@ fn uriencode_maybe(s: &str, yes: bool) -> s3_xml::Value { #[cfg(test)] mod tests { use super::*; - use garage_model::s3::version_table::*; use garage_util::*; use std::iter::FromIterator; @@ -991,10 +982,16 @@ mod tests { ObjectVersion { uuid: Uuid::from(uuid), timestamp: TS, - state: ObjectVersionState::Uploading(ObjectVersionHeaders { - content_type: "text/plain".to_string(), - other: BTreeMap::::new(), - }), + state: ObjectVersionState::Uploading { + multipart: true, + encryption: ObjectVersionEncryption::Plaintext { + inner: ObjectVersionMetaInner { + headers: vec![], + checksum: None, + }, + }, + checksum_algorithm: None, + }, } } @@ -1169,172 +1166,180 @@ mod tests { Ok(()) } - fn version() -> Version { + fn mpu() -> MultipartUpload { let uuid = Uuid::from([0x08; 32]); - let blocks = vec![ + let parts = vec![ ( - VersionBlockKey { + MpuPartKey { part_number: 1, - offset: 1, + timestamp: TS, }, - VersionBlock { - hash: uuid, - size: 3, + MpuPart { + version: uuid, + size: Some(3), + etag: Some("etag1".into()), + checksum: None, }, ), ( - VersionBlockKey { - part_number: 1, - offset: 2, - }, - VersionBlock { - hash: uuid, - size: 2, - }, - ), - ( - VersionBlockKey { + MpuPartKey { part_number: 2, - offset: 1, + timestamp: TS, }, - VersionBlock { - hash: uuid, - size: 8, + MpuPart { + version: uuid, + size: None, + etag: None, + checksum: None, }, ), ( - VersionBlockKey { + MpuPartKey { + part_number: 3, + timestamp: TS, + }, + MpuPart { + version: uuid, + size: Some(10), + etag: Some("etag2".into()), + checksum: None, + }, + ), + ( + MpuPartKey { part_number: 5, - offset: 1, + timestamp: TS, }, - VersionBlock { - hash: uuid, - size: 7, + MpuPart { + version: uuid, + size: Some(7), + etag: Some("etag3".into()), + checksum: None, }, ), ( - VersionBlockKey { + MpuPartKey { part_number: 8, - offset: 1, + timestamp: TS, }, - VersionBlock { - hash: uuid, - size: 5, + MpuPart { + version: uuid, + size: Some(5), + etag: Some("etag4".into()), + checksum: None, }, ), ]; - let etags = vec![ - (1, "etag1".to_string()), - (3, "etag2".to_string()), - (5, "etag3".to_string()), - (8, "etag4".to_string()), - (9, "etag5".to_string()), - ]; - Version { - bucket_id: uuid, - key: "a".to_string(), - uuid, + MultipartUpload { + upload_id: uuid, + timestamp: TS, deleted: false.into(), - blocks: crdt::Map::::from_iter(blocks), - parts_etags: crdt::Map::::from_iter(etags), + parts: crdt::Map::::from_iter(parts), + bucket_id: uuid, + key: "a".into(), } } - fn obj() -> Object { - Object::new(bucket(), "d".to_string(), vec![objup_version([0x08; 32])]) - } - #[test] fn test_fetch_part_info() -> Result<(), Error> { - let uuid = Uuid::from([0x08; 32]); let mut query = ListPartsQuery { bucket_name: "a".to_string(), - bucket_id: uuid, key: "a".to_string(), upload_id: "xx".to_string(), part_number_marker: None, max_parts: 2, }; - assert!( - fetch_part_info(&query, None, None, uuid).is_err(), - "No object and version should fail" - ); - assert!( - fetch_part_info(&query, Some(obj()), None, uuid).is_err(), - "No version should faild" - ); - assert!( - fetch_part_info(&query, None, Some(version()), uuid).is_err(), - "No object should fail" - ); + let mpu = mpu(); // Start from the beginning but with limited size to trigger pagination - let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?; - assert_eq!(pagination.unwrap(), 5); + let (info, pagination) = fetch_part_info(&query, &mpu)?; + assert_eq!(pagination.unwrap(), 3); assert_eq!( info, vec![ PartInfo { - etag: "etag1".to_string(), + etag: "etag1", timestamp: TS, part_number: 1, - size: 5 + size: 3, + checksum: None, }, PartInfo { - etag: "etag3".to_string(), + etag: "etag2", timestamp: TS, - part_number: 5, - size: 7 + part_number: 3, + checksum: None, + size: 10 }, ] ); // Use previous pagination to make a new request query.part_number_marker = Some(pagination.unwrap()); - let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?; + let (info, pagination) = fetch_part_info(&query, &mpu)?; assert!(pagination.is_none()); assert_eq!( info, - vec![PartInfo { - etag: "etag4".to_string(), - timestamp: TS, - part_number: 8, - size: 5 - },] + vec![ + PartInfo { + etag: "etag3", + timestamp: TS, + checksum: None, + part_number: 5, + size: 7 + }, + PartInfo { + etag: "etag4", + timestamp: TS, + checksum: None, + part_number: 8, + size: 5 + }, + ] ); // Trying to access a part that is way larger than registered ones query.part_number_marker = Some(9999); - let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?; + let (info, pagination) = fetch_part_info(&query, &mpu)?; assert!(pagination.is_none()); assert_eq!(info, vec![]); // Try without any limitation query.max_parts = 1000; query.part_number_marker = None; - let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?; + let (info, pagination) = fetch_part_info(&query, &mpu)?; assert!(pagination.is_none()); assert_eq!( info, vec![ PartInfo { - etag: "etag1".to_string(), + etag: "etag1", timestamp: TS, + checksum: None, part_number: 1, - size: 5 + size: 3 }, PartInfo { - etag: "etag3".to_string(), + etag: "etag2", timestamp: TS, + checksum: None, + part_number: 3, + size: 10 + }, + PartInfo { + etag: "etag3", + timestamp: TS, + checksum: None, part_number: 5, size: 7 }, PartInfo { - etag: "etag4".to_string(), + etag: "etag4", timestamp: TS, + checksum: None, part_number: 8, size: 5 }, diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs new file mode 100644 index 00000000..d6eb26cb --- /dev/null +++ b/src/api/s3/multipart.rs @@ -0,0 +1,713 @@ +use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; +use std::hash::Hasher; +use std::sync::Arc; + +use base64::prelude::*; +use crc32c::Crc32cHasher as Crc32c; +use crc32fast::Hasher as Crc32; +use futures::prelude::*; +use hyper::{Request, Response}; +use md5::{Digest, Md5}; +use sha1::Sha1; +use sha2::Sha256; + +use garage_table::*; +use garage_util::data::*; +use garage_util::error::OkOrMessage; + +use garage_model::garage::Garage; +use garage_model::s3::block_ref_table::*; +use garage_model::s3::mpu_table::*; +use garage_model::s3::object_table::*; +use garage_model::s3::version_table::*; + +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::*; +use crate::xml as s3_xml; + +// ---- + +pub async fn handle_create_multipart_upload( + ctx: ReqCtx, + req: &Request, + key: &String, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; + let existing_object = garage.object_table.get(&bucket_id, &key).await?; + + let upload_id = gen_uuid(); + let timestamp = next_timestamp(existing_object.as_ref()); + + let headers = extract_metadata_headers(req.headers())?; + let meta = ObjectVersionMetaInner { + headers, + checksum: None, + }; + + // Determine whether object should be encrypted, and if so the key + let encryption = EncryptionParams::new_from_headers(&garage, req.headers())?; + let object_encryption = encryption.encrypt_meta(meta)?; + + let checksum_algorithm = request_checksum_algorithm(req.headers())?; + + // Create object in object table + let object_version = ObjectVersion { + uuid: upload_id, + timestamp, + state: ObjectVersionState::Uploading { + multipart: true, + encryption: object_encryption, + checksum_algorithm, + }, + }; + let object = Object::new(*bucket_id, key.to_string(), vec![object_version]); + garage.object_table.insert(&object).await?; + + // Create multipart upload in mpu table + // This multipart upload will hold references to uploaded parts + // (which are entries in the Version table) + let mpu = MultipartUpload::new(upload_id, timestamp, *bucket_id, key.into(), false); + garage.mpu_table.insert(&mpu).await?; + + // Send success response + let result = s3_xml::InitiateMultipartUploadResult { + xmlns: (), + bucket: s3_xml::Value(bucket_name.to_string()), + key: s3_xml::Value(key.to_string()), + upload_id: s3_xml::Value(hex::encode(upload_id)), + }; + let xml = s3_xml::to_xml_with_header(&result)?; + + let mut resp = Response::builder(); + encryption.add_response_headers(&mut resp); + Ok(resp.body(string_body(xml))?) +} + +pub async fn handle_put_part( + ctx: ReqCtx, + req: Request, + key: &str, + part_number: u64, + upload_id: &str, +) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + + let upload_id = decode_upload_id(upload_id)?; + + let expected_checksums = ExpectedChecksums { + md5: match req.headers().get("content-md5") { + Some(x) => Some(x.to_str()?.to_string()), + None => None, + }, + sha256: None, + extra: request_checksum_value(req.headers())?, + }; + + let key = key.to_string(); + + let (req_head, mut req_body) = req.into_parts(); + + // Before we stream the body, configure the needed checksums. + req_body.add_expected_checksums(expected_checksums.clone()); + // TODO: avoid parsing encryption headers twice... + if !EncryptionParams::new_from_headers(&garage, &req_head.headers)?.is_encrypted() { + // For non-encrypted objects, we need to compute the md5sum in all cases + // (even if content-md5 is not set), because it is used as an etag of the + // part, which is in turn used in the etag computation of the whole object + req_body.add_md5(); + } + + let (stream, stream_checksums) = req_body.streaming_with_checksums(); + let stream = stream.map_err(Error::from); + + let mut chunker = StreamChunker::new(stream, garage.config.block_size); + + // Read first chuck, and at the same time try to get object to see if it exists + let ((_, object_version, mut mpu), first_block) = + futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?; + + // Check encryption params + let (object_encryption, checksum_algorithm) = match object_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), + _ => unreachable!(), + }; + let (encryption, _) = + EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?; + + // Check object is valid and part can be accepted + let first_block = first_block.ok_or_bad_request("Empty body")?; + + // Calculate part identity: timestamp, version id + let version_uuid = gen_uuid(); + let mpu_part_key = MpuPartKey { + part_number, + timestamp: mpu.next_timestamp(part_number), + }; + + // The following consists in many steps that can each fail. + // Keep track that some cleanup will be needed if things fail + // before everything is finished (cleanup is done using the Drop trait). + let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner { + garage: garage.clone(), + upload_id, + version_uuid, + })); + + // Create version and link version from MPU + mpu.parts.clear(); + mpu.parts.put( + mpu_part_key, + MpuPart { + version: version_uuid, + // all these are filled in later, at the end of this function + etag: None, + checksum: None, + size: None, + }, + ); + garage.mpu_table.insert(&mpu).await?; + + let version = Version::new( + version_uuid, + VersionBacklink::MultipartUpload { upload_id }, + false, + ); + garage.version_table.insert(&version).await?; + + // Copy data to version + let (total_size, _, _) = read_and_put_blocks( + &ctx, + &version, + encryption, + part_number, + first_block, + chunker, + Checksummer::new(), + ) + .await?; + + // Verify that checksums match + let checksums = stream_checksums + .await + .ok_or_internal_error("checksum calculation")??; + + // Store part etag in version + let etag = encryption.etag_from_md5(&checksums.md5); + + mpu.parts.put( + mpu_part_key, + MpuPart { + version: version_uuid, + etag: Some(etag.clone()), + checksum: checksums.extract(checksum_algorithm), + size: Some(total_size), + }, + ); + garage.mpu_table.insert(&mpu).await?; + + // We were not interrupted, everything went fine. + // We won't have to clean up on drop. + interrupted_cleanup.cancel(); + + let mut resp = Response::builder().header("ETag", format!("\"{}\"", etag)); + encryption.add_response_headers(&mut resp); + let resp = add_checksum_response_headers(&expected_checksums.extra, resp); + Ok(resp.body(empty_body())?) +} + +struct InterruptedCleanup(Option); +struct InterruptedCleanupInner { + garage: Arc, + upload_id: Uuid, + version_uuid: Uuid, +} + +impl InterruptedCleanup { + fn cancel(&mut self) { + drop(self.0.take()); + } +} +impl Drop for InterruptedCleanup { + fn drop(&mut self) { + if let Some(info) = self.0.take() { + tokio::spawn(async move { + let version = Version::new( + info.version_uuid, + VersionBacklink::MultipartUpload { + upload_id: info.upload_id, + }, + true, + ); + if let Err(e) = info.garage.version_table.insert(&version).await { + warn!("Cannot cleanup after aborted UploadPart: {}", e); + } + }); + } + } +} + +pub async fn handle_complete_multipart_upload( + ctx: ReqCtx, + req: Request, + key: &str, + upload_id: &str, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; + let (req_head, req_body) = req.into_parts(); + + let expected_checksum = request_checksum_value(&req_head.headers)?; + + let body = req_body.collect().await?; + + let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?; + let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml) + .ok_or_bad_request("Invalid CompleteMultipartUpload XML")?; + debug!( + "CompleteMultipartUpload list of parts: {:?}", + body_list_of_parts + ); + + let upload_id = decode_upload_id(upload_id)?; + + // Get object and multipart upload + let key = key.to_string(); + let (object, mut object_version, mpu) = get_upload(&ctx, &key, &upload_id).await?; + + if mpu.parts.is_empty() { + return Err(Error::bad_request("No data was uploaded")); + } + + let (object_encryption, checksum_algorithm) = match object_version.state { + ObjectVersionState::Uploading { + encryption, + checksum_algorithm, + .. + } => (encryption, checksum_algorithm), + _ => unreachable!(), + }; + + // Check that part numbers are an increasing sequence. + // (it doesn't need to start at 1 nor to be a continuous sequence, + // see discussion in #192) + if body_list_of_parts.is_empty() { + return Err(Error::EntityTooSmall); + } + if !body_list_of_parts + .iter() + .zip(body_list_of_parts.iter().skip(1)) + .all(|(p1, p2)| p1.part_number < p2.part_number) + { + return Err(Error::InvalidPartOrder); + } + + // Check that the list of parts they gave us corresponds to parts we have here + debug!("Parts stored in multipart upload: {:?}", mpu.parts.items()); + let mut have_parts = HashMap::new(); + for (pk, pv) in mpu.parts.items().iter() { + have_parts.insert(pk.part_number, pv); + } + let mut parts = vec![]; + for req_part in body_list_of_parts.iter() { + match have_parts.get(&req_part.part_number) { + Some(part) if part.etag.as_ref() == Some(&req_part.etag) && part.size.is_some() => { + // alternative version: if req_part.checksum.is_some() && part.checksum != req_part.checksum { + if part.checksum != req_part.checksum { + return Err(Error::InvalidDigest(format!( + "Invalid checksum for part {}: in request = {:?}, uploaded part = {:?}", + req_part.part_number, req_part.checksum, part.checksum + ))); + } + parts.push(*part) + } + _ => return Err(Error::InvalidPart), + } + } + + let grg = &garage; + let parts_versions = futures::future::try_join_all(parts.iter().map(|p| async move { + grg.version_table + .get(&p.version, &EmptyKey) + .await? + .ok_or_internal_error("Part version missing from version table") + })) + .await?; + + // Create final version and block refs + let mut final_version = Version::new( + upload_id, + VersionBacklink::Object { + bucket_id: *bucket_id, + key: key.to_string(), + }, + false, + ); + for (part_number, part_version) in parts_versions.iter().enumerate() { + if part_version.deleted.get() { + return Err(Error::InvalidPart); + } + for (vbk, vb) in part_version.blocks.items().iter() { + final_version.blocks.put( + VersionBlockKey { + part_number: (part_number + 1) as u64, + offset: vbk.offset, + }, + *vb, + ); + } + } + garage.version_table.insert(&final_version).await?; + + let block_refs = final_version.blocks.items().iter().map(|(_, b)| BlockRef { + block: b.hash, + version: upload_id, + deleted: false.into(), + }); + garage.block_ref_table.insert_many(block_refs).await?; + + // Calculate checksum and etag of final object + // To understand how etags are calculated, read more here: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + // https://teppen.io/2018/06/23/aws_s3_etags/ + let mut checksummer = MultipartChecksummer::init(checksum_algorithm); + for part in parts.iter() { + checksummer.update(part.etag.as_ref().unwrap(), part.checksum)?; + } + let (checksum_md5, checksum_extra) = checksummer.finalize(); + + if expected_checksum.is_some() && checksum_extra != expected_checksum { + return Err(Error::InvalidDigest( + "Failed to validate x-amz-checksum-*".into(), + )); + } + + let etag = format!("{}-{}", hex::encode(&checksum_md5[..]), parts.len()); + + // Calculate total size of final object + let total_size = parts.iter().map(|x| x.size.unwrap()).sum(); + + if let Err(e) = check_quotas(&ctx, total_size, Some(&object)).await { + object_version.state = ObjectVersionState::Aborted; + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); + garage.object_table.insert(&final_object).await?; + + return Err(e); + } + + // If there is a checksum algorithm, update metadata with checksum + let object_encryption = match checksum_algorithm { + None => object_encryption, + Some(_) => { + let (encryption, meta) = + EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?; + let new_meta = ObjectVersionMetaInner { + headers: meta.into_owned().headers, + checksum: checksum_extra, + }; + encryption.encrypt_meta(new_meta)? + } + }; + + // Write final object version + object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( + ObjectVersionMeta { + encryption: object_encryption, + size: total_size, + etag: etag.clone(), + }, + final_version.blocks.items()[0].1.hash, + )); + + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); + garage.object_table.insert(&final_object).await?; + + // Send response saying ok we're done + let result = s3_xml::CompleteMultipartUploadResult { + xmlns: (), + // FIXME: the location returned is not always correct: + // - we always return https, but maybe some people do http + // - if root_domain is not specified, a full URL is not returned + location: garage + .config + .s3_api + .root_domain + .as_ref() + .map(|rd| s3_xml::Value(format!("https://{}.{}/{}", bucket_name, rd, key))) + .or(Some(s3_xml::Value(format!("/{}/{}", bucket_name, key)))), + bucket: s3_xml::Value(bucket_name.to_string()), + key: s3_xml::Value(key), + etag: s3_xml::Value(format!("\"{}\"", etag)), + checksum_crc32: match &checksum_extra { + Some(ChecksumValue::Crc32(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_crc32c: match &checksum_extra { + Some(ChecksumValue::Crc32c(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_sha1: match &checksum_extra { + Some(ChecksumValue::Sha1(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + checksum_sha256: match &checksum_extra { + Some(ChecksumValue::Sha256(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, + }; + let xml = s3_xml::to_xml_with_header(&result)?; + + let resp = Response::builder(); + let resp = add_checksum_response_headers(&expected_checksum, resp); + Ok(resp.body(string_body(xml))?) +} + +pub async fn handle_abort_multipart_upload( + ctx: ReqCtx, + key: &str, + upload_id: &str, +) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + + let upload_id = decode_upload_id(upload_id)?; + + let (_, mut object_version, _) = get_upload(&ctx, &key.to_string(), &upload_id).await?; + + object_version.state = ObjectVersionState::Aborted; + let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]); + garage.object_table.insert(&final_object).await?; + + Ok(Response::new(empty_body())) +} + +// ======== helpers ============ + +#[allow(clippy::ptr_arg)] +pub(crate) async fn get_upload( + ctx: &ReqCtx, + key: &String, + upload_id: &Uuid, +) -> Result<(Object, ObjectVersion, MultipartUpload), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; + let (object, mpu) = futures::try_join!( + garage.object_table.get(bucket_id, key).map_err(Error::from), + garage + .mpu_table + .get(upload_id, &EmptyKey) + .map_err(Error::from), + )?; + + let object = object.ok_or(Error::NoSuchUpload)?; + let mpu = mpu.ok_or(Error::NoSuchUpload)?; + + let object_version = object + .versions() + .iter() + .find(|v| v.uuid == *upload_id && v.is_uploading(Some(true))) + .ok_or(Error::NoSuchUpload)? + .clone(); + + Ok((object, object_version, mpu)) +} + +pub fn decode_upload_id(id: &str) -> Result { + let id_bin = hex::decode(id).map_err(|_| Error::NoSuchUpload)?; + if id_bin.len() != 32 { + return Err(Error::NoSuchUpload); + } + let mut uuid = [0u8; 32]; + uuid.copy_from_slice(&id_bin[..]); + Ok(Uuid::from(uuid)) +} + +#[derive(Debug)] +struct CompleteMultipartUploadPart { + etag: String, + part_number: u64, + checksum: Option, +} + +fn parse_complete_multipart_upload_body( + xml: &roxmltree::Document, +) -> Option> { + let mut parts = vec![]; + + let root = xml.root(); + let cmu = root.first_child()?; + if !cmu.has_tag_name("CompleteMultipartUpload") { + return None; + } + + for item in cmu.children() { + // Only parse nodes + if !item.is_element() { + continue; + } + + if item.has_tag_name("Part") { + let etag = item.children().find(|e| e.has_tag_name("ETag"))?.text()?; + let part_number = item + .children() + .find(|e| e.has_tag_name("PartNumber"))? + .text()?; + let checksum = if let Some(crc32) = + item.children().find(|e| e.has_tag_name("ChecksumCRC32")) + { + Some(ChecksumValue::Crc32( + BASE64_STANDARD.decode(crc32.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(crc32c) = item.children().find(|e| e.has_tag_name("ChecksumCRC32C")) + { + Some(ChecksumValue::Crc32c( + BASE64_STANDARD.decode(crc32c.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) { + Some(ChecksumValue::Sha1( + BASE64_STANDARD.decode(sha1.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else if let Some(sha256) = item.children().find(|e| e.has_tag_name("ChecksumSHA256")) + { + Some(ChecksumValue::Sha256( + BASE64_STANDARD.decode(sha256.text()?).ok()?[..] + .try_into() + .ok()?, + )) + } else { + None + }; + parts.push(CompleteMultipartUploadPart { + etag: etag.trim_matches('"').to_string(), + part_number: part_number.parse().ok()?, + checksum, + }); + } else { + return None; + } + } + + Some(parts) +} + +// ====== checksummer ==== + +#[derive(Default)] +pub(crate) struct MultipartChecksummer { + pub md5: Md5, + pub extra: Option, +} + +pub(crate) enum MultipartExtraChecksummer { + Crc32(Crc32), + Crc32c(Crc32c), + Sha1(Sha1), + Sha256(Sha256), +} + +impl MultipartChecksummer { + pub(crate) fn init(algo: Option) -> Self { + Self { + md5: Md5::new(), + extra: match algo { + None => None, + Some(ChecksumAlgorithm::Crc32) => { + Some(MultipartExtraChecksummer::Crc32(Crc32::new())) + } + Some(ChecksumAlgorithm::Crc32c) => { + Some(MultipartExtraChecksummer::Crc32c(Crc32c::default())) + } + Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())), + Some(ChecksumAlgorithm::Sha256) => { + Some(MultipartExtraChecksummer::Sha256(Sha256::new())) + } + }, + } + } + + pub(crate) fn update( + &mut self, + etag: &str, + checksum: Option, + ) -> Result<(), Error> { + self.md5 + .update(&hex::decode(&etag).ok_or_message("invalid etag hex")?); + match (&mut self.extra, checksum) { + (None, _) => (), + ( + Some(MultipartExtraChecksummer::Crc32(ref mut crc32)), + Some(ChecksumValue::Crc32(x)), + ) => { + crc32.update(&x); + } + ( + Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)), + Some(ChecksumValue::Crc32c(x)), + ) => { + crc32c.write(&x); + } + (Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => { + sha1.update(&x); + } + ( + Some(MultipartExtraChecksummer::Sha256(ref mut sha256)), + Some(ChecksumValue::Sha256(x)), + ) => { + sha256.update(&x); + } + (Some(_), b) => { + return Err(Error::internal_error(format!( + "part checksum was not computed correctly, got: {:?}", + b + ))) + } + } + Ok(()) + } + + pub(crate) fn finalize(self) -> (Md5Checksum, Option) { + let md5 = self.md5.finalize()[..].try_into().unwrap(); + let extra = match self.extra { + None => None, + Some(MultipartExtraChecksummer::Crc32(crc32)) => { + Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize()))) + } + Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c( + u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()), + )), + Some(MultipartExtraChecksummer::Sha1(sha1)) => { + Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap())) + } + Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256( + sha256.finalize()[..].try_into().unwrap(), + )), + }; + (md5, extra) + } +} diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index f2098ab0..09be7e7c 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::convert::TryInto; +use std::convert::{Infallible, TryInto}; use std::ops::RangeInclusive; use std::sync::Arc; use std::task::{Context, Poll}; @@ -9,28 +9,35 @@ use bytes::Bytes; use chrono::{DateTime, Duration, Utc}; use futures::{Stream, StreamExt}; use hyper::header::{self, HeaderMap, HeaderName, HeaderValue}; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode}; use multer::{Constraints, Multipart, SizeLimit}; use serde::Deserialize; use garage_model::garage::Garage; +use garage_model::s3::object_table::*; -use crate::s3::error::*; -use crate::s3::put::{get_headers, save_stream}; -use crate::s3::xml as s3_xml; -use crate::signature::payload::{parse_date, verify_v4}; +use garage_api_common::cors::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; + +use crate::api_server::ResBody; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::{extract_metadata_headers, save_stream, ChecksumMode}; +use crate::xml as s3_xml; pub async fn handle_post_object( garage: Arc, - req: Request, + req: Request, bucket_name: String, -) -> Result, Error> { +) -> Result, Error> { let boundary = req .headers() .get(header::CONTENT_TYPE) .and_then(|ct| ct.to_str().ok()) .and_then(|ct| multer::parse_boundary(ct).ok()) - .ok_or_bad_request("Counld not get multipart boundary")?; + .ok_or_bad_request("Could not get multipart boundary")?; // 16k seems plenty for a header. 5G is the max size of a single part, so it seems reasonable // for a PostObject @@ -41,16 +48,21 @@ pub async fn handle_post_object( ); let (head, body) = req.into_parts(); - let mut multipart = Multipart::with_constraints(body, boundary, constraints); + let stream = body_stream::<_, Error>(body); + let mut multipart = Multipart::with_constraints(stream, boundary, constraints); let mut params = HeaderMap::new(); - let field = loop { + let file_field = loop { let field = if let Some(field) = multipart.next_field().await? { field } else { return Err(Error::bad_request("Request did not contain a file")); }; - let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) { + let name: HeaderName = if let Some(Ok(name)) = field + .name() + .map(str::to_ascii_lowercase) + .map(TryInto::try_into) + { name } else { continue; @@ -60,23 +72,11 @@ pub async fn handle_post_object( } if let Ok(content) = HeaderValue::from_str(&field.text().await?) { - match name.as_str() { - "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), - "acl" => { - if params.insert("x-amz-acl", content).is_some() { - return Err(Error::bad_request( - "Field 'acl' provided more than one time", - )); - } - } - _ => { - if params.insert(&name, content).is_some() { - return Err(Error::bad_request(format!( - "Field '{}' provided more than one time", - name - ))); - } - } + if params.insert(&name, content).is_some() { + return Err(Error::bad_request(format!( + "Field '{}' provided more than once", + name + ))); } } }; @@ -86,26 +86,15 @@ pub async fn handle_post_object( .get("key") .ok_or_bad_request("No key was provided")? .to_str()?; - let credential = params - .get("x-amz-credential") - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? - .to_str()?; let policy = params .get("policy") .ok_or_bad_request("No policy was provided")? .to_str()?; - let signature = params - .get("x-amz-signature") - .ok_or_bad_request("No signature was provided")? - .to_str()?; - let date = params - .get("x-amz-date") - .ok_or_bad_request("No date was provided")? - .to_str()?; + let authorization = Authorization::parse_form(¶ms)?; let key = if key.contains("${filename}") { // if no filename is provided, don't replace. This matches the behavior of AWS. - if let Some(filename) = field.file_name() { + if let Some(filename) = file_field.file_name() { key.replace("${filename}", filename) } else { key.to_owned() @@ -114,21 +103,13 @@ pub async fn handle_post_object( key.to_owned() }; - let date = parse_date(date)?; - let api_key = verify_v4( - &garage, - "s3", - credential, - &date, - signature, - policy.as_bytes(), - ) - .await?; + let api_key = verify_v4(&garage, "s3", &authorization, policy.as_bytes()).await?; let bucket_id = garage .bucket_helper() .resolve_bucket(&bucket_name, &api_key) - .await?; + .await + .map_err(pass_helper_error)?; if !api_key.allow_write(&bucket_id) { return Err(Error::forbidden("Operation is not allowed for this key.")); @@ -138,6 +119,12 @@ pub async fn handle_post_object( .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); + let matching_cors_rule = find_matching_cors_rule( + &bucket_params, + &Request::from_parts(head.clone(), empty_body::()), + )? + .cloned(); let decoded_policy = BASE64_STANDARD .decode(policy) @@ -149,16 +136,31 @@ pub async fn handle_post_object( .ok_or_bad_request("Invalid expiration date")? .into(); if Utc::now() - expiration > Duration::zero() { - return Err(Error::bad_request("Expiration date is in the paste")); + return Err(Error::bad_request("Expiration date is in the past")); } let mut conditions = decoded_policy.into_conditions()?; + // If there are conditions on the bucket name, check these against the actual bucket_name rather + // than the one in params, which is allowed to be absent. + if let Some(conds) = conditions.params.remove("bucket") { + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s.as_str() == bucket_name, + Operation::StartsWith(s) => bucket_name.starts_with(&s), + }; + if !ok { + return Err(Error::bad_request( + "Key 'bucket' has value not allowed in policy", + )); + } + } + } + for (param_key, value) in params.iter() { - let mut param_key = param_key.to_string(); - param_key.make_ascii_lowercase(); - match param_key.as_str() { - "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields + let param_key = param_key.as_str(); + match param_key { + "policy" | "x-amz-signature" | "bucket" => (), // this is always accepted, as it's required to validate other fields "content-type" => { let conds = conditions.params.remove("content-type").ok_or_else(|| { Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) @@ -202,7 +204,7 @@ pub async fn handle_post_object( // how aws seems to behave. continue; } - let conds = conditions.params.remove(¶m_key).ok_or_else(|| { + let conds = conditions.params.remove(param_key).ok_or_else(|| { Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { @@ -228,23 +230,52 @@ pub async fn handle_post_object( ))); } - let headers = get_headers(¶ms)?; + // if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere + // around here to make sure the rest of the machinery takes our acl into account. + let headers = extract_metadata_headers(¶ms)?; - let stream = field.map(|r| r.map_err(Into::into)); - let (_, md5) = save_stream( - garage, + let checksum_algorithm = request_checksum_algorithm(¶ms)?; + let expected_checksums = ExpectedChecksums { + md5: params + .get("content-md5") + .map(HeaderValue::to_str) + .transpose()? + .map(str::to_string), + sha256: None, + extra: checksum_algorithm + .map(|algo| extract_checksum_value(¶ms, algo)) + .transpose()?, + }; + + let meta = ObjectVersionMetaInner { headers, + checksum: expected_checksums.extra, + }; + + let encryption = EncryptionParams::new_from_headers(&garage, ¶ms)?; + + let stream = file_field.map(|r| r.map_err(Into::into)); + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + + let res = save_stream( + &ctx, + meta, + encryption, StreamLimiter::new(stream, conditions.content_length), - &bucket, &key, - None, - None, + ChecksumMode::Verify(&expected_checksums), ) .await?; - let etag = format!("\"{}\"", md5); + let etag = format!("\"{}\"", res.etag); - let resp = if let Some(mut target) = params + let mut resp = if let Some(mut target) = params .get("success_action_redirect") .and_then(|h| h.to_str().ok()) .and_then(|u| url::Url::parse(u).ok()) @@ -252,20 +283,20 @@ pub async fn handle_post_object( { target .query_pairs_mut() - .append_pair("bucket", &bucket_name) + .append_pair("bucket", &ctx.bucket_name) .append_pair("key", &key) .append_pair("etag", &etag); let target = target.to_string(); - Response::builder() + let mut resp = Response::builder() .status(StatusCode::SEE_OTHER) .header(header::LOCATION, target.clone()) - .header(header::ETAG, etag) - .body(target.into())? + .header(header::ETAG, etag); + encryption.add_response_headers(&mut resp); + resp.body(string_body(target))? } else { let path = head .uri - .into_parts() - .path_and_query + .path_and_query() .map(|paq| paq.path().to_string()) .unwrap_or_else(|| "/".to_string()); let authority = head @@ -288,28 +319,34 @@ pub async fn handle_post_object( .get("success_action_status") .and_then(|h| h.to_str().ok()) .unwrap_or("204"); - let builder = Response::builder() + let mut builder = Response::builder() .header(header::LOCATION, location.clone()) .header(header::ETAG, etag.clone()); + encryption.add_response_headers(&mut builder); match action { - "200" => builder.status(StatusCode::OK).body(Body::empty())?, + "200" => builder.status(StatusCode::OK).body(empty_body())?, "201" => { let xml = s3_xml::PostObject { xmlns: (), location: s3_xml::Value(location), - bucket: s3_xml::Value(bucket_name), + bucket: s3_xml::Value(ctx.bucket_name), key: s3_xml::Value(key), etag: s3_xml::Value(etag), }; let body = s3_xml::to_xml_with_header(&xml)?; builder .status(StatusCode::CREATED) - .body(Body::from(body.into_bytes()))? + .body(string_body(body))? } - _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, + _ => builder.status(StatusCode::NO_CONTENT).body(empty_body())?, } }; + if let Some(rule) = matching_cors_rule { + add_cors_headers(&mut resp, &rule) + .ok_or_internal_error("Invalid bucket CORS configuration")?; + } + Ok(resp) } @@ -330,7 +367,7 @@ impl Policy { if map.len() != 1 { return Err(Error::bad_request("Invalid policy item")); } - let (mut k, v) = map.into_iter().next().expect("size was verified"); + let (mut k, v) = map.into_iter().next().expect("Size could not be verified"); k.make_ascii_lowercase(); params.entry(k).or_default().push(Operation::Equal(v)); } diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 350ab884..b915f2ec 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -1,242 +1,340 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::Arc; -use base64::prelude::*; use futures::prelude::*; -use hyper::body::{Body, Bytes}; +use futures::stream::FuturesOrdered; +use futures::try_join; + +use tokio::sync::mpsc; + +use hyper::body::Bytes; use hyper::header::{HeaderMap, HeaderValue}; use hyper::{Request, Response}; -use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; -use sha2::Sha256; use opentelemetry::{ trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, Context, }; -use garage_rpc::netapp::bytes_buf::BytesBuf; +use garage_net::bytes_buf::BytesBuf; +use garage_rpc::rpc_helper::OrderTag; use garage_table::*; -use garage_util::async_hash::*; use garage_util::data::*; use garage_util::error::Error as GarageError; use garage_util::time::*; use garage_block::manager::INLINE_THRESHOLD; -use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::index_counter::CountedItem; use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use garage_api_common::helpers::*; +use garage_api_common::signature::body::StreamingChecksumReceiver; +use garage_api_common::signature::checksum::*; + +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; + +pub(crate) struct SaveStreamResult { + pub(crate) version_uuid: Uuid, + pub(crate) version_timestamp: u64, + /// Etag WITHOUT THE QUOTES (just the hex value) + pub(crate) etag: String, +} + +pub(crate) enum ChecksumMode<'a> { + Verify(&'a ExpectedChecksums), + VerifyFrom { + checksummer: StreamingChecksumReceiver, + trailer_algo: Option, + }, + Calculate(Option), +} pub async fn handle_put( - garage: Arc, - req: Request, - bucket: &Bucket, - key: &str, - content_sha256: Option, -) -> Result, Error> { + ctx: ReqCtx, + req: Request, + key: &String, +) -> Result, Error> { // Retrieve interesting headers from request - let headers = get_headers(req.headers())?; + let headers = extract_metadata_headers(req.headers())?; debug!("Object headers: {:?}", headers); - let content_md5 = match req.headers().get("content-md5") { - Some(x) => Some(x.to_str()?.to_string()), - None => None, + let expected_checksums = ExpectedChecksums { + md5: match req.headers().get("content-md5") { + Some(x) => Some(x.to_str()?.to_string()), + None => None, + }, + sha256: None, + extra: request_checksum_value(req.headers())?, + }; + let trailer_checksum_algorithm = request_trailer_checksum_algorithm(req.headers())?; + + let meta = ObjectVersionMetaInner { + headers, + checksum: expected_checksums.extra, }; - let (_head, body) = req.into_parts(); - let body = body.map_err(Error::from); + // Determine whether object should be encrypted, and if so the key + let encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?; - save_stream( - garage, - headers, - body, - bucket, + // The request body is a special ReqBody object (see garage_api_common::signature::body) + // which supports calculating checksums while streaming the data. + // Before we start streaming, we configure it to calculate all the checksums we need. + let mut req_body = req.into_body(); + req_body.add_expected_checksums(expected_checksums.clone()); + if !encryption.is_encrypted() { + // For non-encrypted objects, we need to compute the md5sum in all cases + // (even if content-md5 is not set), because it is used as the object etag + req_body.add_md5(); + } + + let (stream, checksummer) = req_body.streaming_with_checksums(); + let stream = stream.map_err(Error::from); + + let res = save_stream( + &ctx, + meta, + encryption, + stream, key, - content_md5, - content_sha256, + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo: trailer_checksum_algorithm, + }, ) - .await - .map(|(uuid, md5)| put_response(uuid, md5)) + .await?; + + let mut resp = Response::builder() + .header("x-amz-version-id", hex::encode(res.version_uuid)) + .header("ETag", format!("\"{}\"", res.etag)); + encryption.add_response_headers(&mut resp); + let resp = add_checksum_response_headers(&expected_checksums.extra, resp); + Ok(resp.body(empty_body())?) } pub(crate) async fn save_stream> + Unpin>( - garage: Arc, - headers: ObjectVersionHeaders, + ctx: &ReqCtx, + mut meta: ObjectVersionMetaInner, + encryption: EncryptionParams, body: S, - bucket: &Bucket, - key: &str, - content_md5: Option, - content_sha256: Option, -) -> Result<(Uuid, String), Error> { - // Generate identity of new version - let version_uuid = gen_uuid(); - let version_timestamp = now_msec(); + key: &String, + checksum_mode: ChecksumMode<'_>, +) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let mut chunker = StreamChunker::new(body, garage.config.block_size); - let first_block = chunker.next().await?.unwrap_or_default(); + let (first_block_opt, existing_object) = try_join!( + chunker.next(), + garage.object_table.get(bucket_id, key).map_err(Error::from), + )?; + + let first_block = first_block_opt.unwrap_or_default(); + + // Generate identity of new version + let version_uuid = gen_uuid(); + let version_timestamp = next_timestamp(existing_object.as_ref()); + + let mut checksummer = match &checksum_mode { + ChecksumMode::Verify(expected) => Checksummer::init(expected, !encryption.is_encrypted()), + ChecksumMode::Calculate(algo) => { + Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(*algo) + } + ChecksumMode::VerifyFrom { .. } => { + // Checksums are calculated by the garage_api_common::signature module + // so here we can just have an empty checksummer that does nothing + Checksummer::new() + } + }; // If body is small enough, store it directly in the object table // as "inline data". We can then return immediately. if first_block.len() < INLINE_THRESHOLD { - let mut md5sum = Md5::new(); - md5sum.update(&first_block[..]); - let data_md5sum = md5sum.finalize(); - let data_md5sum_hex = hex::encode(data_md5sum); + checksummer.update(&first_block); + let mut checksums = checksummer.finalize(); + + match checksum_mode { + ChecksumMode::Verify(expected) => { + checksums.verify(&expected)?; + } + ChecksumMode::Calculate(algo) => { + meta.checksum = checksums.extract(algo); + } + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo, + } => { + drop(chunker); + checksums = checksummer + .await + .ok_or_internal_error("checksum calculation")??; + if let Some(algo) = trailer_algo { + meta.checksum = checksums.extract(Some(algo)); + } + } + }; - let data_sha256sum = sha256sum(&first_block[..]); let size = first_block.len() as u64; + check_quotas(ctx, size, existing_object.as_ref()).await?; - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; - - check_quotas(&garage, bucket, key, size).await?; + let etag = encryption.etag_from_md5(&checksums.md5); + let inline_data = encryption.encrypt_blob(&first_block)?.to_vec(); let object_version = ObjectVersion { uuid: version_uuid, timestamp: version_timestamp, state: ObjectVersionState::Complete(ObjectVersionData::Inline( ObjectVersionMeta { - headers, + encryption: encryption.encrypt_meta(meta)?, size, - etag: data_md5sum_hex.clone(), + etag: etag.clone(), }, - first_block.to_vec(), + inline_data, )), }; - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - return Ok((version_uuid, data_md5sum_hex)); + return Ok(SaveStreamResult { + version_uuid, + version_timestamp, + etag, + }); } // The following consists in many steps that can each fail. // Keep track that some cleanup will be needed if things fail // before everything is finished (cleanup is done using the Drop trait). - let mut interrupted_cleanup = InterruptedCleanup(Some(( - garage.clone(), - bucket.id, - key.into(), + let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner { + garage: garage.clone(), + bucket_id: *bucket_id, + key: key.into(), version_uuid, version_timestamp, - ))); + })); // Write version identifier in object table so that we have a trace // that we are uploading something let mut object_version = ObjectVersion { uuid: version_uuid, timestamp: version_timestamp, - state: ObjectVersionState::Uploading(headers.clone()), + state: ObjectVersionState::Uploading { + encryption: encryption.encrypt_meta(meta.clone())?, + checksum_algorithm: None, // don't care; overwritten later + multipart: false, + }, }; - let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]); + let object = Object::new(*bucket_id, key.into(), vec![object_version.clone()]); garage.object_table.insert(&object).await?; // Initialize corresponding entry in version table // Write this entry now, even with empty block list, // to prevent block_ref entries from being deleted (they can be deleted // if the reference a version that isn't found in the version table) - let version = Version::new(version_uuid, bucket.id, key.into(), false); + let version = Version::new( + version_uuid, + VersionBacklink::Object { + bucket_id: *bucket_id, + key: key.into(), + }, + false, + ); garage.version_table.insert(&version).await?; - // Transfer data and verify checksum - let first_block_hash = async_blake2sum(first_block.clone()).await; - - let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks( - &garage, + // Transfer data + let (total_size, mut checksums, first_block_hash) = read_and_put_blocks( + ctx, &version, + encryption, 1, first_block, - first_block_hash, - &mut chunker, + chunker, + checksummer, ) .await?; - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; + // Verify checksums are ok / add calculated checksum to metadata + match checksum_mode { + ChecksumMode::Verify(expected) => { + checksums.verify(&expected)?; + } + ChecksumMode::Calculate(algo) => { + meta.checksum = checksums.extract(algo); + } + ChecksumMode::VerifyFrom { + checksummer, + trailer_algo, + } => { + checksums = checksummer + .await + .ok_or_internal_error("checksum calculation")??; + if let Some(algo) = trailer_algo { + meta.checksum = checksums.extract(Some(algo)); + } + } + }; - check_quotas(&garage, bucket, key, total_size).await?; + // Verify quotas are respsected + check_quotas(ctx, total_size, existing_object.as_ref()).await?; // Save final object state, marked as Complete - let md5sum_hex = hex::encode(data_md5sum); + let etag = encryption.etag_from_md5(&checksums.md5); + object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( ObjectVersionMeta { - headers, + encryption: encryption.encrypt_meta(meta)?, size: total_size, - etag: md5sum_hex.clone(), + etag: etag.clone(), }, first_block_hash, )); - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; // We were not interrupted, everything went fine. // We won't have to clean up on drop. interrupted_cleanup.cancel(); - Ok((version_uuid, md5sum_hex)) -} - -/// Validate MD5 sum against content-md5 header -/// and sha256sum against signed content-sha256 -fn ensure_checksum_matches( - data_md5sum: &[u8], - data_sha256sum: garage_util::data::FixedBytes32, - content_md5: Option<&str>, - content_sha256: Option, -) -> Result<(), Error> { - if let Some(expected_sha256) = content_sha256 { - if expected_sha256 != data_sha256sum { - return Err(Error::bad_request( - "Unable to validate x-amz-content-sha256", - )); - } else { - trace!("Successfully validated x-amz-content-sha256"); - } - } - if let Some(expected_md5) = content_md5 { - if expected_md5.trim_matches('"') != BASE64_STANDARD.encode(data_md5sum) { - return Err(Error::bad_request("Unable to validate content-md5")); - } else { - trace!("Successfully validated content-md5"); - } - } - Ok(()) + Ok(SaveStreamResult { + version_uuid, + version_timestamp, + etag, + }) } /// Check that inserting this object with this size doesn't exceed bucket quotas -async fn check_quotas( - garage: &Arc, - bucket: &Bucket, - key: &str, +pub(crate) async fn check_quotas( + ctx: &ReqCtx, size: u64, + prev_object: Option<&Object>, ) -> Result<(), Error> { - let quotas = bucket.state.as_option().unwrap().quotas.get(); + let ReqCtx { + garage, + bucket_id, + bucket_params, + .. + } = ctx; + + let quotas = bucket_params.quotas.get(); if quotas.max_objects.is_none() && quotas.max_size.is_none() { return Ok(()); }; - let key = key.to_string(); - let (prev_object, counters) = futures::try_join!( - garage.object_table.get(&bucket.id, &key), - garage.object_counter_table.table.get(&bucket.id, &EmptyKey), - )?; + let counters = garage + .object_counter_table + .table + .get(bucket_id, &EmptyKey) + .await?; let counters = counters - .map(|x| x.filtered_values(&garage.system.ring.borrow())) + .map(|x| x.filtered_values(&garage.system.cluster_layout())) .unwrap_or_default(); let (prev_cnt_obj, prev_cnt_size) = match prev_object { @@ -267,7 +365,7 @@ async fn check_quotas( if cnt_size_diff > 0 && current_size + cnt_size_diff > ms as i64 { return Err(Error::forbidden(format!( "Bucket size quota is reached, maximum total size of objects for this bucket: {}. The bucket is already {} bytes, and this object would add {} bytes.", - ms, current_size, size + ms, current_size, cnt_size_diff ))); } } @@ -275,90 +373,186 @@ async fn check_quotas( Ok(()) } -async fn read_and_put_blocks> + Unpin>( - garage: &Garage, +pub(crate) async fn read_and_put_blocks> + Unpin>( + ctx: &ReqCtx, version: &Version, + encryption: EncryptionParams, part_number: u64, first_block: Bytes, - first_block_hash: Hash, - chunker: &mut StreamChunker, -) -> Result<(u64, GenericArray, Hash), Error> { + mut chunker: StreamChunker, + checksummer: Checksummer, +) -> Result<(u64, Checksums, Hash), Error> { let tracer = opentelemetry::global::tracer("garage"); - let md5hasher = AsyncHasher::::new(); - let sha256hasher = AsyncHasher::::new(); + let (block_tx, mut block_rx) = mpsc::channel::>(2); + let read_blocks = async { + block_tx.send(Ok(first_block)).await?; + loop { + let res = chunker + .next() + .with_context(Context::current_with_span( + tracer.start("Read block from client"), + )) + .await; + match res { + Ok(Some(block)) => block_tx.send(Ok(block)).await?, + Ok(None) => break, + Err(e) => { + block_tx.send(Err(e)).await?; + break; + } + } + } + drop(block_tx); + Ok::<_, mpsc::error::SendError<_>>(()) + }; - futures::future::join( - md5hasher.update(first_block.clone()), - sha256hasher.update(first_block.clone()), - ) - .with_context(Context::current_with_span( - tracer.start("Hash first block (md5, sha256)"), - )) - .await; + let (block_tx2, mut block_rx2) = mpsc::channel::>(1); + let hash_stream = async { + let mut checksummer = checksummer; + while let Some(next) = block_rx.recv().await { + match next { + Ok(block) => { + block_tx2.send(Ok(block.clone())).await?; + checksummer = tokio::task::spawn_blocking(move || { + checksummer.update(&block); + checksummer + }) + .with_context(Context::current_with_span( + tracer.start("Hash block (md5, sha256)"), + )) + .await + .unwrap() + } + Err(e) => { + block_tx2.send(Err(e)).await?; + break; + } + } + } + drop(block_tx2); + Ok::<_, mpsc::error::SendError<_>>(checksummer) + }; - let mut next_offset = first_block.len(); - let mut put_curr_version_block = put_block_meta( - garage, - version, - part_number, - 0, - first_block_hash, - first_block.len() as u64, - ); - let mut put_curr_block = garage - .block_manager - .rpc_put_block(first_block_hash, first_block); + let (block_tx3, mut block_rx3) = mpsc::channel::>(1); + let encrypt_hash_blocks = async { + let mut first_block_hash = None; + while let Some(next) = block_rx2.recv().await { + match next { + Ok(block) => { + let unencrypted_len = block.len() as u64; + let res = tokio::task::spawn_blocking(move || { + let block = encryption.encrypt_block(block)?; + let hash = blake2sum(&block); + Ok((block, hash)) + }) + .with_context(Context::current_with_span( + tracer.start("Encrypt and hash (blake2) block"), + )) + .await + .unwrap(); + match res { + Ok((block, hash)) => { + if first_block_hash.is_none() { + first_block_hash = Some(hash); + } + block_tx3.send(Ok((block, unencrypted_len, hash))).await?; + } + Err(e) => { + block_tx3.send(Err(e)).await?; + break; + } + } + } + Err(e) => { + block_tx3.send(Err(e)).await?; + break; + } + } + } + drop(block_tx3); + Ok::<_, mpsc::error::SendError<_>>(first_block_hash.unwrap()) + }; - loop { - let (_, _, next_block) = futures::try_join!( - put_curr_block.map_err(Error::from), - put_curr_version_block.map_err(Error::from), - chunker.next(), - )?; - if let Some(block) = next_block { - let (_, _, block_hash) = futures::future::join3( - md5hasher.update(block.clone()), - sha256hasher.update(block.clone()), - async_blake2sum(block.clone()), - ) - .with_context(Context::current_with_span( - tracer.start("Hash block (md5, sha256, blake2)"), - )) - .await; - let block_len = block.len(); - put_curr_version_block = put_block_meta( - garage, + let put_blocks = async { + // Structure for handling several concurrent writes to storage nodes + let order_stream = OrderTag::stream(); + let mut write_futs = FuturesOrdered::new(); + let mut written_bytes = 0u64; + loop { + // Simultaneously write blocks to storage nodes & await for next block to be written + let currently_running = write_futs.len(); + let write_futs_next = async { + if write_futs.is_empty() { + futures::future::pending().await + } else { + write_futs.next().await.unwrap() + } + }; + let recv_next = async { + // If more than a maximum number of writes are in progress, don't add more for now + if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request { + futures::future::pending().await + } else { + block_rx3.recv().await + } + }; + let (block, unencrypted_len, hash) = tokio::select! { + result = write_futs_next => { + result?; + continue; + }, + recv = recv_next => match recv { + Some(next) => next?, + None => break, + }, + }; + + // For next block to be written: count its size and spawn future to write it + write_futs.push_back(put_block_and_meta( + ctx, version, part_number, - next_offset as u64, - block_hash, - block_len as u64, - ); - put_curr_block = garage.block_manager.rpc_put_block(block_hash, block); - next_offset += block_len; - } else { - break; + written_bytes, + hash, + block, + unencrypted_len, + encryption.is_encrypted(), + order_stream.order(written_bytes), + )); + written_bytes += unencrypted_len; } - } + while let Some(res) = write_futs.next().await { + res?; + } + Ok::<_, Error>(written_bytes) + }; - let total_size = next_offset as u64; - let data_md5sum = md5hasher.finalize().await; + let (_, stream_hash_result, block_hash_result, final_result) = + futures::join!(read_blocks, hash_stream, encrypt_hash_blocks, put_blocks); - let data_sha256sum = sha256hasher.finalize().await; - let data_sha256sum = Hash::try_from(&data_sha256sum[..]).unwrap(); + let total_size = final_result?; + // unwrap here is ok, because if hasher failed, it is because something failed + // later in the pipeline which already caused a return at the ? on previous line + let first_block_hash = block_hash_result.unwrap(); + let checksums = stream_hash_result.unwrap().finalize(); - Ok((total_size, data_md5sum, data_sha256sum)) + Ok((total_size, checksums, first_block_hash)) } -async fn put_block_meta( - garage: &Garage, +async fn put_block_and_meta( + ctx: &ReqCtx, version: &Version, part_number: u64, offset: u64, hash: Hash, + block: Bytes, size: u64, + is_encrypted: bool, + order_tag: OrderTag, ) -> Result<(), GarageError> { + let ReqCtx { garage, .. } = ctx; + let mut version = version.clone(); version.blocks.put( VersionBlockKey { @@ -375,13 +569,16 @@ async fn put_block_meta( }; futures::try_join!( + garage + .block_manager + .rpc_put_block(hash, block, is_encrypted, Some(order_tag)), garage.version_table.insert(&version), garage.block_ref_table.insert(&block_ref), )?; Ok(()) } -struct StreamChunker>> { +pub(crate) struct StreamChunker>> { stream: S, read_all: bool, block_size: usize, @@ -389,7 +586,7 @@ struct StreamChunker>> { } impl> + Unpin> StreamChunker { - fn new(stream: S, block_size: usize) -> Self { + pub(crate) fn new(stream: S, block_size: usize) -> Self { Self { stream, read_all: false, @@ -398,7 +595,7 @@ impl> + Unpin> StreamChunker { } } - async fn next(&mut self) -> Result, Error> { + pub(crate) async fn next(&mut self) -> Result, Error> { while !self.read_all && self.buf.len() < self.block_size { if let Some(block) = self.stream.next().await { let bytes = block?; @@ -417,16 +614,15 @@ impl> + Unpin> StreamChunker { } } -pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response { - Response::builder() - .header("x-amz-version-id", hex::encode(version_uuid)) - .header("ETag", format!("\"{}\"", md5sum_hex)) - .body(Body::from(vec![])) - .unwrap() +struct InterruptedCleanup(Option); +struct InterruptedCleanupInner { + garage: Arc, + bucket_id: Uuid, + key: String, + version_uuid: Uuid, + version_timestamp: u64, } -struct InterruptedCleanup(Option<(Arc, Uuid, String, Uuid, u64)>); - impl InterruptedCleanup { fn cancel(&mut self) { drop(self.0.take()); @@ -434,15 +630,15 @@ impl InterruptedCleanup { } impl Drop for InterruptedCleanup { fn drop(&mut self) { - if let Some((garage, bucket_id, key, version_uuid, version_ts)) = self.0.take() { + if let Some(info) = self.0.take() { tokio::spawn(async move { let object_version = ObjectVersion { - uuid: version_uuid, - timestamp: version_ts, + uuid: info.version_uuid, + timestamp: info.version_timestamp, state: ObjectVersionState::Aborted, }; - let object = Object::new(bucket_id, key, vec![object_version]); - if let Err(e) = garage.object_table.insert(&object).await { + let object = Object::new(info.bucket_id, info.key, vec![object_version]); + if let Err(e) = info.garage.object_table.insert(&object).await { warn!("Cannot cleanup after aborted PutObject: {}", e); } }); @@ -450,425 +646,57 @@ impl Drop for InterruptedCleanup { } } -// ---- +// ============ helpers ============ -pub async fn handle_create_multipart_upload( - garage: Arc, - req: &Request, - bucket_name: &str, - bucket_id: Uuid, - key: &str, -) -> Result, Error> { - let version_uuid = gen_uuid(); - let headers = get_headers(req.headers())?; - - // Create object in object table - let object_version = ObjectVersion { - uuid: version_uuid, - timestamp: now_msec(), - state: ObjectVersionState::Uploading(headers), - }; - let object = Object::new(bucket_id, key.to_string(), vec![object_version]); - garage.object_table.insert(&object).await?; - - // Insert empty version so that block_ref entries refer to something - // (they are inserted concurrently with blocks in the version table, so - // there is the possibility that they are inserted before the version table - // is created, in which case it is allowed to delete them, e.g. in repair_*) - let version = Version::new(version_uuid, bucket_id, key.into(), false); - garage.version_table.insert(&version).await?; - - // Send success response - let result = s3_xml::InitiateMultipartUploadResult { - xmlns: (), - bucket: s3_xml::Value(bucket_name.to_string()), - key: s3_xml::Value(key.to_string()), - upload_id: s3_xml::Value(hex::encode(version_uuid)), - }; - let xml = s3_xml::to_xml_with_header(&result)?; - - Ok(Response::new(Body::from(xml.into_bytes()))) -} - -pub async fn handle_put_part( - garage: Arc, - req: Request, - bucket_id: Uuid, - key: &str, - part_number: u64, - upload_id: &str, - content_sha256: Option, -) -> Result, Error> { - let version_uuid = decode_upload_id(upload_id)?; - - let content_md5 = match req.headers().get("content-md5") { - Some(x) => Some(x.to_str()?.to_string()), - None => None, - }; - - // Read first chuck, and at the same time try to get object to see if it exists - let key = key.to_string(); - - let body = req.into_body().map_err(Error::from); - let mut chunker = StreamChunker::new(body, garage.config.block_size); - - let (object, version, first_block) = futures::try_join!( - garage - .object_table - .get(&bucket_id, &key) - .map_err(Error::from), - garage - .version_table - .get(&version_uuid, &EmptyKey) - .map_err(Error::from), - chunker.next(), - )?; - - // Check object is valid and multipart block can be accepted - let first_block = first_block.ok_or_bad_request("Empty body")?; - let object = object.ok_or_bad_request("Object not found")?; - - if !object - .versions() - .iter() - .any(|v| v.uuid == version_uuid && v.is_uploading()) - { - return Err(Error::NoSuchUpload); - } - - // Check part hasn't already been uploaded - if let Some(v) = version { - if v.has_part_number(part_number) { - return Err(Error::bad_request(format!( - "Part number {} has already been uploaded", - part_number - ))); - } - } - - // Copy block to store - let version = Version::new(version_uuid, bucket_id, key, false); - - let first_block_hash = async_blake2sum(first_block.clone()).await; - - let (_, data_md5sum, data_sha256sum) = read_and_put_blocks( - &garage, - &version, - part_number, - first_block, - first_block_hash, - &mut chunker, - ) - .await?; - - // Verify that checksums map - ensure_checksum_matches( - data_md5sum.as_slice(), - data_sha256sum, - content_md5.as_deref(), - content_sha256, - )?; - - // Store part etag in version - let data_md5sum_hex = hex::encode(data_md5sum); - let mut version = version; - version - .parts_etags - .put(part_number, data_md5sum_hex.clone()); - garage.version_table.insert(&version).await?; - - let response = Response::builder() - .header("ETag", format!("\"{}\"", data_md5sum_hex)) - .body(Body::empty()) - .unwrap(); - Ok(response) -} - -pub async fn handle_complete_multipart_upload( - garage: Arc, - req: Request, - bucket_name: &str, - bucket: &Bucket, - key: &str, - upload_id: &str, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?; - let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml) - .ok_or_bad_request("Invalid CompleteMultipartUpload XML")?; - debug!( - "CompleteMultipartUpload list of parts: {:?}", - body_list_of_parts - ); - - let version_uuid = decode_upload_id(upload_id)?; - - // Get object and version - let key = key.to_string(); - let (object, version) = futures::try_join!( - garage.object_table.get(&bucket.id, &key), - garage.version_table.get(&version_uuid, &EmptyKey), - )?; - - let object = object.ok_or(Error::NoSuchKey)?; - let mut object_version = object - .versions() - .iter() - .find(|v| v.uuid == version_uuid && v.is_uploading()) - .cloned() - .ok_or(Error::NoSuchUpload)?; - - let version = version.ok_or(Error::NoSuchKey)?; - if version.blocks.is_empty() { - return Err(Error::bad_request("No data was uploaded")); - } - - let headers = match object_version.state { - ObjectVersionState::Uploading(headers) => headers, - _ => unreachable!(), - }; - - // Check that part numbers are an increasing sequence. - // (it doesn't need to start at 1 nor to be a continuous sequence, - // see discussion in #192) - if body_list_of_parts.is_empty() { - return Err(Error::EntityTooSmall); - } - if !body_list_of_parts - .iter() - .zip(body_list_of_parts.iter().skip(1)) - .all(|(p1, p2)| p1.part_number < p2.part_number) - { - return Err(Error::InvalidPartOrder); - } - - // Garage-specific restriction, see #204: part numbers must be - // consecutive starting at 1 - if body_list_of_parts[0].part_number != 1 - || !body_list_of_parts - .iter() - .zip(body_list_of_parts.iter().skip(1)) - .all(|(p1, p2)| p1.part_number + 1 == p2.part_number) - { - return Err(Error::NotImplemented("Garage does not support completing a Multipart upload with non-consecutive part numbers. This is a restriction of Garage's data model, which might be fixed in a future release. See issue #204 for more information on this topic.".into())); - } - - // Check that the list of parts they gave us corresponds to the parts we have here - debug!("Expected parts from request: {:?}", body_list_of_parts); - debug!("Parts stored in version: {:?}", version.parts_etags.items()); - let parts = version - .parts_etags - .items() - .iter() - .map(|pair| (&pair.0, &pair.1)); - let same_parts = body_list_of_parts - .iter() - .map(|x| (&x.part_number, &x.etag)) - .eq(parts); - if !same_parts { - return Err(Error::InvalidPart); - } - - // Check that all blocks belong to one of the parts - let block_parts = version - .blocks - .items() - .iter() - .map(|(bk, _)| bk.part_number) - .collect::>(); - let same_parts = body_list_of_parts - .iter() - .map(|x| x.part_number) - .eq(block_parts.into_iter()); - if !same_parts { - return Err(Error::bad_request( - "Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again." - )); - } - - // Calculate etag of final object - // To understand how etags are calculated, read more here: - // https://teppen.io/2018/06/23/aws_s3_etags/ - let num_parts = body_list_of_parts.len(); - let mut etag_md5_hasher = Md5::new(); - for (_, etag) in version.parts_etags.items().iter() { - etag_md5_hasher.update(etag.as_bytes()); - } - let etag = format!("{}-{}", hex::encode(etag_md5_hasher.finalize()), num_parts); - - // Calculate total size of final object - let total_size = version.blocks.items().iter().map(|x| x.1.size).sum(); - - if let Err(e) = check_quotas(&garage, bucket, &key, total_size).await { - object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); - garage.object_table.insert(&final_object).await?; - - return Err(e); - } - - // Write final object version - object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( - ObjectVersionMeta { - headers, - size: total_size, - etag: etag.clone(), - }, - version.blocks.items()[0].1.hash, - )); - - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); - garage.object_table.insert(&final_object).await?; - - // Send response saying ok we're done - let result = s3_xml::CompleteMultipartUploadResult { - xmlns: (), - location: None, - bucket: s3_xml::Value(bucket_name.to_string()), - key: s3_xml::Value(key), - etag: s3_xml::Value(format!("\"{}\"", etag)), - }; - let xml = s3_xml::to_xml_with_header(&result)?; - - Ok(Response::new(Body::from(xml.into_bytes()))) -} - -pub async fn handle_abort_multipart_upload( - garage: Arc, - bucket_id: Uuid, - key: &str, - upload_id: &str, -) -> Result, Error> { - let version_uuid = decode_upload_id(upload_id)?; - - let object = garage - .object_table - .get(&bucket_id, &key.to_string()) - .await?; - let object = object.ok_or(Error::NoSuchKey)?; - - let object_version = object - .versions() - .iter() - .find(|v| v.uuid == version_uuid && v.is_uploading()); - let mut object_version = match object_version { - None => return Err(Error::NoSuchUpload), - Some(x) => x.clone(), - }; - - object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket_id, key.to_string(), vec![object_version]); - garage.object_table.insert(&final_object).await?; - - Ok(Response::new(Body::from(vec![]))) -} - -fn get_mime_type(headers: &HeaderMap) -> Result { - Ok(headers - .get(hyper::header::CONTENT_TYPE) - .map(|x| x.to_str()) - .unwrap_or(Ok("blob"))? - .to_string()) -} - -pub(crate) fn get_headers(headers: &HeaderMap) -> Result { - let content_type = get_mime_type(headers)?; - let mut other = BTreeMap::new(); +pub(crate) fn extract_metadata_headers( + headers: &HeaderMap, +) -> Result { + let mut ret = Vec::new(); // Preserve standard headers let standard_header = vec![ + hyper::header::CONTENT_TYPE, hyper::header::CACHE_CONTROL, hyper::header::CONTENT_DISPOSITION, hyper::header::CONTENT_ENCODING, hyper::header::CONTENT_LANGUAGE, hyper::header::EXPIRES, ]; - for h in standard_header.iter() { - if let Some(v) = headers.get(h) { - match v.to_str() { - Ok(v_str) => { - other.insert(h.to_string(), v_str.to_string()); - } - Err(e) => { - warn!("Discarding header {}, error in .to_str(): {}", h, e); - } - } + for name in standard_header.iter() { + if let Some(value) = headers.get(name) { + ret.push((name.to_string(), value.to_str()?.to_string())); } } // Preserve x-amz-meta- headers - for (k, v) in headers.iter() { - if k.as_str().starts_with("x-amz-meta-") { - match v.to_str() { - Ok(v_str) => { - other.insert(k.to_string(), v_str.to_string()); - } - Err(e) => { - warn!("Discarding header {}, error in .to_str(): {}", k, e); - } + for (name, value) in headers.iter() { + if name.as_str().starts_with("x-amz-meta-") { + ret.push(( + name.as_str().to_ascii_lowercase(), + std::str::from_utf8(value.as_bytes())?.to_string(), + )); + } + if name == X_AMZ_WEBSITE_REDIRECT_LOCATION { + let value = std::str::from_utf8(value.as_bytes())?.to_string(); + if !(value.starts_with("/") + || value.starts_with("http://") + || value.starts_with("https://")) + { + return Err(Error::bad_request(format!( + "Invalid {X_AMZ_WEBSITE_REDIRECT_LOCATION} header", + ))); } + ret.push((X_AMZ_WEBSITE_REDIRECT_LOCATION.to_string(), value)); } } - Ok(ObjectVersionHeaders { - content_type, - other, - }) + Ok(ret) } -pub fn decode_upload_id(id: &str) -> Result { - let id_bin = hex::decode(id).map_err(|_| Error::NoSuchUpload)?; - if id_bin.len() != 32 { - return Err(Error::NoSuchUpload); - } - let mut uuid = [0u8; 32]; - uuid.copy_from_slice(&id_bin[..]); - Ok(Uuid::from(uuid)) -} - -#[derive(Debug)] -struct CompleteMultipartUploadPart { - etag: String, - part_number: u64, -} - -fn parse_complete_multipart_upload_body( - xml: &roxmltree::Document, -) -> Option> { - let mut parts = vec![]; - - let root = xml.root(); - let cmu = root.first_child()?; - if !cmu.has_tag_name("CompleteMultipartUpload") { - return None; - } - - for item in cmu.children() { - // Only parse nodes - if !item.is_element() { - continue; - } - - if item.has_tag_name("Part") { - let etag = item.children().find(|e| e.has_tag_name("ETag"))?.text()?; - let part_number = item - .children() - .find(|e| e.has_tag_name("PartNumber"))? - .text()?; - parts.push(CompleteMultipartUploadPart { - etag: etag.trim_matches('"').to_string(), - part_number: part_number.parse().ok()?, - }); - } else { - return None; - } - } - - Some(parts) +pub(crate) fn next_timestamp(existing_object: Option<&Object>) -> u64 { + existing_object + .as_ref() + .and_then(|obj| obj.versions().iter().map(|v| v.timestamp).max()) + .map(|t| std::cmp::max(t + 1, now_msec())) + .unwrap_or_else(now_msec) } diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 821b0e07..e3f58490 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,9 +3,10 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; -use crate::s3::error::*; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; + +use crate::error::*; router_match! {@func @@ -125,6 +126,12 @@ pub enum Endpoint { key: String, part_number: Option, version_id: Option, + response_cache_control: Option, + response_content_disposition: Option, + response_content_encoding: Option, + response_content_language: Option, + response_content_type: Option, + response_expires: Option, }, GetObjectAcl { key: String, @@ -170,7 +177,7 @@ pub enum Endpoint { }, ListBuckets, ListMultipartUploads { - delimiter: Option, + delimiter: Option, encoding_type: Option, key_marker: Option, max_uploads: Option, @@ -178,7 +185,7 @@ pub enum Endpoint { upload_id_marker: Option, }, ListObjects { - delimiter: Option, + delimiter: Option, encoding_type: Option, marker: Option, max_keys: Option, @@ -188,7 +195,7 @@ pub enum Endpoint { // This value should always be 2. It is not checked when constructing the struct list_type: String, continuation_token: Option, - delimiter: Option, + delimiter: Option, encoding_type: Option, fetch_owner: Option, max_keys: Option, @@ -196,7 +203,7 @@ pub enum Endpoint { start_after: Option, }, ListObjectVersions { - delimiter: Option, + delimiter: Option, encoding_type: Option, key_marker: Option, max_keys: Option, @@ -345,6 +352,18 @@ impl Endpoint { _ => return Err(Error::bad_request("Unknown method")), }; + if let Some(x_id) = query.x_id.take() { + if x_id != res.name() { + // I think AWS ignores the x-id parameter. + // Let's make this at least be a warnin to help debugging. + warn!( + "x-id ({}) does not match parsed endpoint ({})", + x_id, + res.name() + ); + } + } + if let Some(message) = query.nonempty_message() { debug!("Unused query parameter: {}", message) } @@ -358,7 +377,14 @@ impl Endpoint { (query.keyword.take().unwrap_or_default(), key, query, None), key: [ EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker), - EMPTY => GetObject (query_opt::version_id, opt_parse::part_number), + EMPTY => GetObject (query_opt::version_id, + opt_parse::part_number, + query_opt::response_cache_control, + query_opt::response_content_disposition, + query_opt::response_content_encoding, + query_opt::response_content_language, + query_opt::response_content_type, + query_opt::response_expires), ACL => GetObjectAcl (query_opt::version_id), LEGAL_HOLD => GetObjectLegalHold (query_opt::version_id), RETENTION => GetObjectRetention (query_opt::version_id), @@ -671,12 +697,19 @@ generateQueryParameters! { "partNumber" => part_number, "part-number-marker" => part_number_marker, "prefix" => prefix, + "response-cache-control" => response_cache_control, + "response-content-disposition" => response_content_disposition, + "response-content-encoding" => response_content_encoding, + "response-content-language" => response_content_language, + "response-content-type" => response_content_type, + "response-expires" => response_expires, "select-type" => select_type, "start-after" => start_after, "uploadId" => upload_id, "upload-id-marker" => upload_id_marker, "versionId" => version_id, - "version-id-marker" => version_id_marker + "version-id-marker" => version_id_marker, + "x-id" => x_id ] } diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 77738971..03cc01d8 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -1,23 +1,22 @@ use quick_xml::de::from_reader; -use std::sync::Arc; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{header::HeaderName, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; - use garage_model::bucket_table::*; -use garage_model::garage::Garage; -use garage_util::data::*; -pub async fn handle_get_website(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +use garage_api_common::helpers::*; - if let Some(website) = param.website_config.get() { +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; + +pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName = + HeaderName::from_static("x-amz-website-redirect-location"); + +pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(website) = bucket_params.website_config.get() { let wc = WebsiteConfiguration { xmlns: (), error_document: website.error_document.as_ref().map(|v| Key { @@ -33,63 +32,59 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result, Error Ok(Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "application/xml") - .body(Body::from(xml))?) + .body(string_body(xml))?) } else { Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } } -pub async fn handle_delete_website( - garage: Arc, - bucket_id: Uuid, -) -> Result, Error> { - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) +pub async fn handle_delete_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.website_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) .await?; - let param = bucket.params_mut().unwrap(); - - param.website_config.update(None); - garage.bucket_table.insert(&bucket).await?; - Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .body(Body::empty())?) + .body(empty_body())?) } pub async fn handle_put_website( - garage: Arc, - bucket_id: Uuid, - req: Request, - content_sha256: Option, -) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; + ctx: ReqCtx, + req: Request, +) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; - if let Some(content_sha256) = content_sha256 { - verify_signed_content(content_sha256, &body[..])?; - } - - let mut bucket = garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - - let param = bucket.params_mut().unwrap(); + let body = req.into_body().collect().await?; let conf: WebsiteConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .website_config .update(Some(conf.into_garage_website_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) - .body(Body::empty())?) + .body(empty_body())?) } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] @@ -277,7 +272,7 @@ impl Redirect { return Err(Error::bad_request("Bad XML: invalid protocol")); } } - // TODO there are probably more invalide cases, but which ones? + // TODO there are probably more invalid cases, but which ones? Ok(()) } } diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 06f11288..fdb36318 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,7 +1,7 @@ use quick_xml::se::to_string; use serde::{Deserialize, Serialize, Serializer}; -use crate::s3::error::Error as ApiError; +use crate::error::Error as ApiError; pub fn to_xml_with_header(x: &T) -> Result { let mut xml = r#""#.to_string(); @@ -13,6 +13,10 @@ pub fn xmlns_tag(_v: &(), s: S) -> Result { s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") } +pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { + s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] pub struct Value(#[serde(rename = "$value")] pub String); @@ -131,6 +135,14 @@ pub struct CompleteMultipartUploadResult { pub key: Value, #[serde(rename = "ETag")] pub etag: Value, + #[serde(rename = "ChecksumCRC32")] + pub checksum_crc32: Option, + #[serde(rename = "ChecksumCRC32C")] + pub checksum_crc32c: Option, + #[serde(rename = "ChecksumSHA1")] + pub checksum_sha1: Option, + #[serde(rename = "ChecksumSHA256")] + pub checksum_sha256: Option, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -197,6 +209,14 @@ pub struct PartItem { pub part_number: IntValue, #[serde(rename = "Size")] pub size: IntValue, + #[serde(rename = "ChecksumCRC32")] + pub checksum_crc32: Option, + #[serde(rename = "ChecksumCRC32C")] + pub checksum_crc32c: Option, + #[serde(rename = "ChecksumSHA1")] + pub checksum_sha1: Option, + #[serde(rename = "ChecksumSHA256")] + pub checksum_sha256: Option, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -303,6 +323,42 @@ pub struct PostObject { pub etag: Value, } +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grantee { + #[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")] + pub xmlns_xsi: (), + #[serde(rename = "xsi:type")] + pub typ: String, + #[serde(rename = "DisplayName")] + pub display_name: Option, + #[serde(rename = "ID")] + pub id: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grant { + #[serde(rename = "Grantee")] + pub grantee: Grantee, + #[serde(rename = "Permission")] + pub permission: Value, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlList { + #[serde(rename = "Grant")] + pub entries: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlPolicy { + #[serde(serialize_with = "xmlns_tag")] + pub xmlns: (), + #[serde(rename = "Owner")] + pub owner: Option, + #[serde(rename = "AccessControlList")] + pub acl: AccessControlList, +} + #[cfg(test)] mod tests { use super::*; @@ -411,6 +467,43 @@ mod tests { Ok(()) } + #[test] + fn get_bucket_acl_result() -> Result<(), ApiError> { + let grant = Grant { + grantee: Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(Value("owner_name".to_string())), + id: Some(Value("qsdfjklm".to_string())), + }, + permission: Value("FULL_CONTROL".to_string()), + }; + + let get_bucket_acl = AccessControlPolicy { + xmlns: (), + owner: None, + acl: AccessControlList { + entries: vec![grant], + }, + }; + assert_eq!( + to_xml_with_header(&get_bucket_acl)?, + "\ +\ + \ + \ + \ + owner_name\ + qsdfjklm\ + \ + FULL_CONTROL\ + \ + \ +" + ); + Ok(()) + } + #[test] fn delete_result() -> Result<(), ApiError> { let delete_result = DeleteResult { @@ -500,6 +593,10 @@ mod tests { bucket: Value("mybucket".to_string()), key: Value("a/plop".to_string()), etag: Value("\"3858f62230ac3c915f300c664312c11f-9\"".to_string()), + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: Some(Value("ZJAnHyG8PeKz9tI8UTcHrJos39A=".into())), + checksum_sha256: None, }; assert_eq!( to_xml_with_header(&result)?, @@ -509,6 +606,7 @@ mod tests { mybucket\ a/plop\ "3858f62230ac3c915f300c664312c11f-9"\ + ZJAnHyG8PeKz9tI8UTcHrJos39A=\ " ); Ok(()) @@ -780,12 +878,22 @@ mod tests { last_modified: Value("2010-11-10T20:48:34.000Z".to_string()), part_number: IntValue(2), size: IntValue(10485760), + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha256: Some(Value( + "5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=".into(), + )), + checksum_sha1: None, }, PartItem { etag: Value("\"aaaa18db4cc2f85cedef654fccc4a4x8\"".to_string()), last_modified: Value("2010-11-10T20:48:33.000Z".to_string()), part_number: IntValue(3), size: IntValue(10485760), + checksum_sha256: None, + checksum_crc32c: None, + checksum_crc32: Some(Value("ZJAnHyG8=".into())), + checksum_sha1: None, }, ], initiator: Initiator { @@ -820,12 +928,14 @@ mod tests { 2010-11-10T20:48:34.000Z\ 2\ 10485760\ + 5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=\ \ \ "aaaa18db4cc2f85cedef654fccc4a4x8"\ 2010-11-10T20:48:33.000Z\ 3\ 10485760\ + ZJAnHyG8=\ \ \ umat-user-11116a31-17b5-4fb7-9df5-b288870f11xx\ diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs deleted file mode 100644 index 4b8b990f..00000000 --- a/src/api/signature/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -use chrono::{DateTime, Utc}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -use garage_util::data::{sha256sum, Hash}; - -pub mod error; -pub mod payload; -pub mod streaming; - -use error::*; - -pub const SHORT_DATE: &str = "%Y%m%d"; -pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; - -type HmacSha256 = Hmac; - -pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { - if expected_sha256 != sha256sum(body) { - return Err(Error::bad_request( - "Request content hash does not match signed hash".to_string(), - )); - } - Ok(()) -} - -pub fn signing_hmac( - datetime: &DateTime, - secret_key: &str, - region: &str, - service: &str, -) -> Result { - let secret = String::from("AWS4") + secret_key; - let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; - date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); - let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; - region_hmac.update(region.as_bytes()); - let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; - service_hmac.update(service.as_bytes()); - let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; - signing_hmac.update(b"aws4_request"); - let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; - Ok(hmac) -} - -pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { - format!( - "{}/{}/{}/aws4_request", - datetime.format(SHORT_DATE), - region, - service - ) -} diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs deleted file mode 100644 index 4c7934e5..00000000 --- a/src/api/signature/payload.rs +++ /dev/null @@ -1,323 +0,0 @@ -use std::collections::HashMap; - -use chrono::{DateTime, Duration, NaiveDateTime, Utc}; -use hmac::Mac; -use hyper::{Body, Method, Request}; -use sha2::{Digest, Sha256}; - -use garage_table::*; -use garage_util::data::Hash; - -use garage_model::garage::Garage; -use garage_model::key_table::*; - -use super::LONG_DATETIME; -use super::{compute_scope, signing_hmac}; - -use crate::encoding::uri_encode; -use crate::signature::error::*; - -pub async fn check_payload_signature( - garage: &Garage, - service: &str, - request: &Request, -) -> Result<(Option, Option), Error> { - let mut headers = HashMap::new(); - for (key, val) in request.headers() { - headers.insert(key.to_string(), val.to_str()?.to_string()); - } - if let Some(query) = request.uri().query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - for (key, val) in query_pairs { - headers.insert(key.to_lowercase(), val.to_string()); - } - } - - let authorization = if let Some(authorization) = headers.get("authorization") { - parse_authorization(authorization, &headers)? - } else if let Some(algorithm) = headers.get("x-amz-algorithm") { - parse_query_authorization(algorithm, &headers)? - } else { - let content_sha256 = headers.get("x-amz-content-sha256"); - if let Some(content_sha256) = content_sha256.filter(|c| "UNSIGNED-PAYLOAD" != c.as_str()) { - let sha256 = hex::decode(content_sha256) - .ok() - .and_then(|bytes| Hash::try_from(&bytes)) - .ok_or_bad_request("Invalid content sha256 hash")?; - return Ok((None, Some(sha256))); - } else { - return Ok((None, None)); - } - }; - - let canonical_request = canonical_request( - request.method(), - request.uri(), - &headers, - &authorization.signed_headers, - &authorization.content_sha256, - ); - let (_, scope) = parse_credential(&authorization.credential)?; - let string_to_sign = string_to_sign(&authorization.date, &scope, &canonical_request); - - trace!("canonical request:\n{}", canonical_request); - trace!("string to sign:\n{}", string_to_sign); - - let key = verify_v4( - garage, - service, - &authorization.credential, - &authorization.date, - &authorization.signature, - string_to_sign.as_bytes(), - ) - .await?; - - let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" { - None - } else if authorization.content_sha256 == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" { - let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?) - } else { - let bytes = hex::decode(authorization.content_sha256) - .ok_or_bad_request("Invalid content sha256 hash")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?) - }; - - Ok((Some(key), content_sha256)) -} - -struct Authorization { - credential: String, - signed_headers: String, - signature: String, - content_sha256: String, - date: DateTime, -} - -fn parse_authorization( - authorization: &str, - headers: &HashMap, -) -> Result { - let first_space = authorization - .find(' ') - .ok_or_bad_request("Authorization field to short")?; - let (auth_kind, rest) = authorization.split_at(first_space); - - if auth_kind != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request("Unsupported authorization method")); - } - - let mut auth_params = HashMap::new(); - for auth_part in rest.split(',') { - let auth_part = auth_part.trim(); - let eq = auth_part - .find('=') - .ok_or_bad_request("Field without value in authorization header")?; - let (key, value) = auth_part.split_at(eq); - auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); - } - - let cred = auth_params - .get("Credential") - .ok_or_bad_request("Could not find Credential in Authorization field")?; - - let content_sha256 = headers - .get("x-amz-content-sha256") - .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; - - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; - - if Utc::now() - date > Duration::hours(24) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - let auth = Authorization { - credential: cred.to_string(), - signed_headers: auth_params - .get("SignedHeaders") - .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? - .to_string(), - signature: auth_params - .get("Signature") - .ok_or_bad_request("Could not find Signature in Authorization field")? - .to_string(), - content_sha256: content_sha256.to_string(), - date, - }; - Ok(auth) -} - -fn parse_query_authorization( - algorithm: &str, - headers: &HashMap, -) -> Result { - if algorithm != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request( - "Unsupported authorization method".to_string(), - )); - } - - let cred = headers - .get("x-amz-credential") - .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; - let signed_headers = headers - .get("x-amz-signedheaders") - .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; - let signature = headers - .get("x-amz-signature") - .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; - let content_sha256 = headers - .get("x-amz-content-sha256") - .map(|x| x.as_str()) - .unwrap_or("UNSIGNED-PAYLOAD"); - - let duration = headers - .get("x-amz-expires") - .ok_or_bad_request("X-Amz-Expires not found in query parameters")? - .parse() - .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; - - if duration > 7 * 24 * 3600 { - return Err(Error::bad_request( - "X-Amz-Exprires may not exceed a week".to_string(), - )); - } - - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; - - if Utc::now() - date > Duration::seconds(duration) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - Ok(Authorization { - credential: cred.to_string(), - signed_headers: signed_headers.to_string(), - signature: signature.to_string(), - content_sha256: content_sha256.to_string(), - date, - }) -} - -fn parse_credential(cred: &str) -> Result<(String, String), Error> { - let first_slash = cred - .find('/') - .ok_or_bad_request("Credentials does not contain / in authorization field")?; - let (key_id, scope) = cred.split_at(first_slash); - Ok(( - key_id.to_string(), - scope.trim_start_matches('/').to_string(), - )) -} - -pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { - let mut hasher = Sha256::default(); - hasher.update(canonical_req.as_bytes()); - [ - "AWS4-HMAC-SHA256", - &datetime.format(LONG_DATETIME).to_string(), - scope_string, - &hex::encode(hasher.finalize().as_slice()), - ] - .join("\n") -} - -pub fn canonical_request( - method: &Method, - uri: &hyper::Uri, - headers: &HashMap, - signed_headers: &str, - content_sha256: &str, -) -> String { - [ - method.as_str(), - uri.path(), - &canonical_query_string(uri), - &canonical_header_string(headers, signed_headers), - "", - signed_headers, - content_sha256, - ] - .join("\n") -} - -fn canonical_header_string(headers: &HashMap, signed_headers: &str) -> String { - let signed_headers_vec = signed_headers.split(';').collect::>(); - let mut items = headers - .iter() - .filter(|(key, _)| signed_headers_vec.contains(&key.as_str())) - .collect::>(); - items.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); - items - .iter() - .map(|(key, value)| key.to_lowercase() + ":" + value.trim()) - .collect::>() - .join("\n") -} - -fn canonical_query_string(uri: &hyper::Uri) -> String { - if let Some(query) = uri.query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - let mut items = query_pairs - .filter(|(key, _)| key != "X-Amz-Signature") - .map(|(key, value)| uri_encode(&key, true) + "=" + &uri_encode(&value, true)) - .collect::>(); - items.sort(); - items.join("&") - } else { - "".to_string() - } -} - -pub fn parse_date(date: &str) -> Result, Error> { - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - Ok(DateTime::from_utc(date, Utc)) -} - -pub async fn verify_v4( - garage: &Garage, - service: &str, - credential: &str, - date: &DateTime, - signature: &str, - payload: &[u8], -) -> Result { - let (key_id, scope) = parse_credential(credential)?; - - let scope_expected = compute_scope(date, &garage.config.s3_api.s3_region, service); - if scope != scope_expected { - return Err(Error::AuthorizationHeaderMalformed(scope.to_string())); - } - - let key = garage - .key_table - .get(&EmptyKey, &key_id) - .await? - .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::forbidden(format!("No such key: {}", &key_id)))?; - let key_p = key.params().unwrap(); - - let mut hmac = signing_hmac( - date, - &key_p.secret_key, - &garage.config.s3_api.s3_region, - service, - ) - .ok_or_internal_error("Unable to build signing HMAC")?; - hmac.update(payload); - let our_signature = hex::encode(hmac.finalize().into_bytes()); - if signature != our_signature { - return Err(Error::forbidden("Invalid signature".to_string())); - } - - Ok(key) -} diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs deleted file mode 100644 index c8358c4f..00000000 --- a/src/api/signature/streaming.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::pin::Pin; - -use chrono::{DateTime, NaiveDateTime, Utc}; -use futures::prelude::*; -use futures::task; -use garage_model::key_table::Key; -use hmac::Mac; -use hyper::body::Bytes; -use hyper::{Body, Request}; - -use garage_util::data::Hash; - -use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; - -use crate::signature::error::*; - -pub fn parse_streaming_body( - api_key: &Key, - req: Request, - content_sha256: &mut Option, - region: &str, - service: &str, -) -> Result, Error> { - match req.headers().get("x-amz-content-sha256") { - Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => { - let signature = content_sha256 - .take() - .ok_or_bad_request("No signature provided")?; - - let secret_key = &api_key - .state - .as_option() - .ok_or_internal_error("Deleted key state")? - .secret_key; - - let date = req - .headers() - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field")? - .to_str()?; - let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) - .ok_or_bad_request("Invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); - - let scope = compute_scope(&date, region, service); - let signing_hmac = crate::signature::signing_hmac(&date, secret_key, region, service) - .ok_or_internal_error("Unable to build signing HMAC")?; - - Ok(req.map(move |body| { - Body::wrap_stream( - SignedPayloadStream::new( - body.map_err(Error::from), - signing_hmac, - date, - &scope, - signature, - ) - .map_err(Error::from), - ) - })) - } - _ => Ok(req), - } -} - -/// Result of `sha256("")` -const EMPTY_STRING_HEX_DIGEST: &str = - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - -fn compute_streaming_payload_signature( - signing_hmac: &HmacSha256, - date: DateTime, - scope: &str, - previous_signature: Hash, - content_sha256: Hash, -) -> Result { - let string_to_sign = [ - "AWS4-HMAC-SHA256-PAYLOAD", - &date.format(LONG_DATETIME).to_string(), - scope, - &hex::encode(previous_signature), - EMPTY_STRING_HEX_DIGEST, - &hex::encode(content_sha256), - ] - .join("\n"); - - let mut hmac = signing_hmac.clone(); - hmac.update(string_to_sign.as_bytes()); - - Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?) -} - -mod payload { - use garage_util::data::Hash; - - pub enum Error { - Parser(nom::error::Error), - BadSignature, - } - - impl Error { - pub fn description(&self) -> &str { - match *self { - Error::Parser(ref e) => e.code.description(), - Error::BadSignature => "Bad signature", - } - } - } - - #[derive(Debug, Clone)] - pub struct Header { - pub size: usize, - pub signature: Hash, - } - - impl Header { - pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { - use nom::bytes::streaming::tag; - use nom::character::streaming::hex_digit1; - use nom::combinator::map_res; - use nom::number::streaming::hex_u32; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(|e| e.map(Error::Parser))? - }; - } - - let (input, size) = try_parse!(hex_u32(input)); - let (input, _) = try_parse!(tag(";")(input)); - - let (input, _) = try_parse!(tag("chunk-signature=")(input)); - let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); - let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; - - let (input, _) = try_parse!(tag("\r\n")(input)); - - let header = Header { - size: size as usize, - signature, - }; - - Ok((input, header)) - } - } -} - -#[derive(Debug)] -pub enum SignedPayloadStreamError { - Stream(Error), - InvalidSignature, - Message(String), -} - -impl SignedPayloadStreamError { - fn message(msg: &str) -> Self { - SignedPayloadStreamError::Message(msg.into()) - } -} - -impl From for Error { - fn from(err: SignedPayloadStreamError) -> Self { - match err { - SignedPayloadStreamError::Stream(e) => e, - SignedPayloadStreamError::InvalidSignature => { - Error::bad_request("Invalid payload signature") - } - SignedPayloadStreamError::Message(e) => { - Error::bad_request(format!("Chunk format error: {}", e)) - } - } - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: payload::Error) -> Self { - Self::message(err.description()) - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: nom::error::Error) -> Self { - Self::message(err.code.description()) - } -} - -struct SignedPayload { - header: payload::Header, - data: Bytes, -} - -#[pin_project::pin_project] -pub struct SignedPayloadStream -where - S: Stream>, -{ - #[pin] - stream: S, - buf: bytes::BytesMut, - datetime: DateTime, - scope: String, - signing_hmac: HmacSha256, - previous_signature: Hash, -} - -impl SignedPayloadStream -where - S: Stream>, -{ - pub fn new( - stream: S, - signing_hmac: HmacSha256, - datetime: DateTime, - scope: &str, - seed_signature: Hash, - ) -> Self { - Self { - stream, - buf: bytes::BytesMut::new(), - datetime, - scope: scope.into(), - signing_hmac, - previous_signature: seed_signature, - } - } - - fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> { - use nom::bytes::streaming::{tag, take}; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(nom::Err::convert)? - }; - } - - let (input, header) = try_parse!(payload::Header::parse(input)); - - // 0-sized chunk is the last - if header.size == 0 { - return Ok(( - input, - SignedPayload { - header, - data: Bytes::new(), - }, - )); - } - - let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); - let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); - - let data = Bytes::from(data.to_vec()); - - Ok((input, SignedPayload { header, data })) - } -} - -impl Stream for SignedPayloadStream -where - S: Stream> + Unpin, -{ - type Item = Result; - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut task::Context<'_>, - ) -> task::Poll> { - use std::task::Poll; - - let mut this = self.project(); - - loop { - let (input, payload) = match Self::parse_next(this.buf) { - Ok(res) => res, - Err(nom::Err::Incomplete(_)) => { - match futures::ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(bytes)) => { - this.buf.extend(bytes); - continue; - } - Some(Err(e)) => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e)))) - } - None => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::message( - "Unexpected EOF", - )))); - } - } - } - Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { - return Poll::Ready(Some(Err(e))) - } - }; - - // 0-sized chunk is the last - if payload.data.is_empty() { - return Poll::Ready(None); - } - - let data_sha256sum = sha256sum(&payload.data); - - let expected_signature = compute_streaming_payload_signature( - this.signing_hmac, - *this.datetime, - this.scope, - *this.previous_signature, - data_sha256sum, - ) - .map_err(|e| { - SignedPayloadStreamError::Message(format!("Could not build signature: {}", e)) - })?; - - if payload.header.signature != expected_signature { - return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature))); - } - - *this.buf = input.into(); - *this.previous_signature = payload.header.signature; - - return Poll::Ready(Some(Ok(payload.data))); - } - } - - fn size_hint(&self) -> (usize, Option) { - self.stream.size_hint() - } -} - -#[cfg(test)] -mod tests { - use futures::prelude::*; - - use super::{SignedPayloadStream, SignedPayloadStreamError}; - - #[tokio::test] - async fn test_interrupted_signed_payload_stream() { - use chrono::{DateTime, Utc}; - - use garage_util::data::Hash; - - let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 - .unwrap() - .with_timezone(&Utc); - let secret_key = "test"; - let region = "test"; - let scope = crate::signature::compute_scope(&datetime, region, "s3"); - let signing_hmac = - crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); - - let data: &[&[u8]] = &[b"1"]; - let body = futures::stream::iter(data.iter().map(|block| Ok(block.as_ref().into()))); - - let seed_signature = Hash::default(); - - let mut stream = - SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature); - - assert!(stream.try_next().await.is_err()); - match stream.try_next().await { - Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {} - item => panic!( - "Unexpected result, expected early EOF error, got {:?}", - item - ), - } - } -} diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index c6985754..c4dbba44 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,30 +14,30 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.2", path = "../db" } -garage_rpc = { version = "0.8.2", path = "../rpc" } -garage_util = { version = "0.8.2", path = "../util" } -garage_table = { version = "0.8.2", path = "../table" } +garage_db.workspace = true +garage_net.workspace = true +garage_rpc.workspace = true +garage_util.workspace = true +garage_table.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true -arc-swap = "1.5" -async-trait = "0.1.7" -bytes = "1.0" -hex = "0.4" -tracing = "0.1" -rand = "0.8" +arc-swap.workspace = true +async-trait.workspace = true +bytes.workspace = true +bytesize.workspace = true +hex.workspace = true +tracing.workspace = true +rand.workspace = true -async-compression = { version = "0.3", features = ["tokio", "zstd"] } -zstd = { version = "0.12", default-features = false } +async-compression.workspace = true +zstd.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-util = { version = "0.6", features = ["io"] } +futures.workspace = true +tokio.workspace = true +tokio-util.workspace = true [features] system-libs = [ "zstd/pkg-config" ] diff --git a/src/block/block.rs b/src/block/block.rs index 935aa900..bd95680e 100644 --- a/src/block/block.rs +++ b/src/block/block.rs @@ -1,101 +1,102 @@ +use std::path::PathBuf; + use bytes::Bytes; use serde::{Deserialize, Serialize}; -use zstd::stream::{decode_all as zstd_decode, Encoder}; +use zstd::stream::Encoder; use garage_util::data::*; use garage_util::error::*; +use garage_net::stream::ByteStream; + #[derive(Debug, Serialize, Deserialize, Copy, Clone)] pub enum DataBlockHeader { Plain, Compressed, } -/// A possibly compressed block of data -pub enum DataBlock { - /// Uncompressed data - Plain(Bytes), - /// Data compressed with zstd - Compressed(Bytes), +#[derive(Debug)] +pub struct DataBlockElem { + header: DataBlockHeader, + elem: T, } -impl DataBlock { - /// Query whether this block is compressed +/// A possibly compressed block of data +pub type DataBlock = DataBlockElem; + +/// A path to a possibly compressed block of data +pub type DataBlockPath = DataBlockElem; + +/// A stream of possibly compressed block data +pub type DataBlockStream = DataBlockElem; + +impl DataBlockHeader { pub fn is_compressed(&self) -> bool { - matches!(self, DataBlock::Compressed(_)) + matches!(self, DataBlockHeader::Compressed) + } +} + +impl DataBlockElem { + pub fn from_parts(header: DataBlockHeader, elem: T) -> Self { + Self { header, elem } } - /// Get the inner, possibly compressed buffer. You should probably use [`DataBlock::verify_get`] - /// instead - pub fn inner_buffer(&self) -> &[u8] { - use DataBlock::*; - let (Plain(ref res) | Compressed(ref res)) = self; - res - } - - /// Get the buffer, possibly decompressing it, and verify it's integrity. - /// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system - /// is used instead. - pub fn verify_get(self, hash: Hash) -> Result { - match self { - DataBlock::Plain(data) => { - if blake2sum(&data) == hash { - Ok(data) - } else { - Err(Error::CorruptData(hash)) - } - } - DataBlock::Compressed(data) => zstd_decode(&data[..]) - .map_err(|_| Error::CorruptData(hash)) - .map(Bytes::from), + pub fn plain(elem: T) -> Self { + Self { + header: DataBlockHeader::Plain, + elem, } } - /// Verify data integrity. Allocate less than [`DataBlock::verify_get`] and don't consume self, but - /// does not return the buffer content. + pub fn compressed(elem: T) -> Self { + Self { + header: DataBlockHeader::Compressed, + elem, + } + } + + pub fn into_parts(self) -> (DataBlockHeader, T) { + (self.header, self.elem) + } + + pub fn as_parts_ref(&self) -> (DataBlockHeader, &T) { + (self.header, &self.elem) + } +} + +impl DataBlock { + /// Verify data integrity. Does not return the buffer content. pub fn verify(&self, hash: Hash) -> Result<(), Error> { - match self { - DataBlock::Plain(data) => { - if blake2sum(data) == hash { + match self.header { + DataBlockHeader::Plain => { + if blake2sum(&self.elem) == hash { Ok(()) } else { Err(Error::CorruptData(hash)) } } - DataBlock::Compressed(data) => zstd::stream::copy_decode(&data[..], std::io::sink()) - .map_err(|_| Error::CorruptData(hash)), + DataBlockHeader::Compressed => { + zstd::stream::copy_decode(&self.elem[..], std::io::sink()) + .map_err(|_| Error::CorruptData(hash)) + } } } pub async fn from_buffer(data: Bytes, level: Option) -> DataBlock { tokio::task::spawn_blocking(move || { if let Some(level) = level { - if let Ok(data) = zstd_encode(&data[..], level) { - return DataBlock::Compressed(data.into()); + if let Ok(data_compressed) = zstd_encode(&data[..], level) { + return DataBlock::compressed(data_compressed.into()); } } - DataBlock::Plain(data) + DataBlock::plain(data.into()) }) .await .unwrap() } - - pub fn into_parts(self) -> (DataBlockHeader, Bytes) { - match self { - DataBlock::Plain(data) => (DataBlockHeader::Plain, data), - DataBlock::Compressed(data) => (DataBlockHeader::Compressed, data), - } - } - - pub fn from_parts(h: DataBlockHeader, bytes: Bytes) -> Self { - match h { - DataBlockHeader::Plain => DataBlock::Plain(bytes), - DataBlockHeader::Compressed => DataBlock::Compressed(bytes), - } - } } -fn zstd_encode(mut source: R, level: i32) -> std::io::Result> { +pub fn zstd_encode(mut source: R, level: i32) -> std::io::Result> { let mut result = Vec::::new(); let mut encoder = Encoder::new(&mut result, level)?; encoder.include_checksum(true)?; diff --git a/src/block/layout.rs b/src/block/layout.rs new file mode 100644 index 00000000..00e3debb --- /dev/null +++ b/src/block/layout.rs @@ -0,0 +1,374 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +use garage_util::config::DataDirEnum; +use garage_util::data::Hash; +use garage_util::error::{Error, OkOrMessage}; +use garage_util::migrate::*; + +type Idx = u16; + +const DRIVE_NPART: usize = 1024; + +const HASH_DRIVE_BYTES: (usize, usize) = (2, 3); + +const MARKER_FILE_NAME: &str = "garage-marker"; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct DataLayout { + pub(crate) data_dirs: Vec, + markers: HashMap, + + /// Primary storage location (index in data_dirs) for each partition + /// = the location where the data is supposed to be, blocks are always + /// written there (copies in other dirs may be deleted if they exist) + pub(crate) part_prim: Vec, + /// Secondary storage locations for each partition = locations + /// where data blocks might be, we check from these dirs when reading + pub(crate) part_sec: Vec>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +pub(crate) struct DataDir { + pub(crate) path: PathBuf, + pub(crate) state: DataDirState, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum DataDirState { + Active { capacity: u64 }, + ReadOnly, +} + +impl DataLayout { + pub(crate) fn initialize(dirs: &DataDirEnum) -> Result { + let data_dirs = make_data_dirs(dirs)?; + + // Split partitions proportionnally to capacity for all drives + // to affect primary storage location + let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::(); + assert!(total_cap > 0); + + let mut part_prim = Vec::with_capacity(DRIVE_NPART); + let mut cum_cap = 0; + for (i, dd) in data_dirs.iter().enumerate() { + if let DataDirState::Active { capacity } = dd.state { + cum_cap += capacity; + let n_total = (cum_cap * DRIVE_NPART as u64) / total_cap; + part_prim.resize(n_total as usize, i as Idx); + } + } + assert_eq!(cum_cap, total_cap); + assert_eq!(part_prim.len(), DRIVE_NPART); + + // If any of the storage locations is non-empty, it probably existed before + // this algorithm was added, so add it as a secondary storage location for all partitions + // to make sure existing files are not lost + let mut part_sec = vec![vec![]; DRIVE_NPART]; + for (i, dd) in data_dirs.iter().enumerate() { + if dir_not_empty(&dd.path)? { + for (sec, prim) in part_sec.iter_mut().zip(part_prim.iter()) { + if *prim != i as Idx { + sec.push(i as Idx); + } + } + } + } + + Ok(Self { + data_dirs, + markers: HashMap::new(), + part_prim, + part_sec, + }) + } + + pub(crate) fn update(self, dirs: &DataDirEnum) -> Result { + // Make list of new data directories, exit if nothing changed + let data_dirs = make_data_dirs(dirs)?; + if data_dirs == self.data_dirs { + return Ok(self); + } + + let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::(); + assert!(total_cap > 0); + + // Compute mapping of old indices to new indices + let old2new = self + .data_dirs + .iter() + .map(|x| { + data_dirs + .iter() + .position(|y| y.path == x.path) + .map(|x| x as Idx) + }) + .collect::>(); + + // Compute secondary location list for partitions based on existing + // folders, translating indices from old to new + let mut part_sec = self + .part_sec + .iter() + .map(|dl| { + dl.iter() + .filter_map(|old| old2new.get(*old as usize).copied().flatten()) + .collect::>() + }) + .collect::>(); + + // Compute a vector that, for each data dir, + // contains the list of partitions primarily stored on that drive + let mut dir_prim = vec![vec![]; data_dirs.len()]; + for (ipart, prim) in self.part_prim.iter().enumerate() { + if let Some(new) = old2new.get(*prim as usize).copied().flatten() { + dir_prim[new as usize].push(ipart); + } + } + + // Compute the target number of partitions per data directory + let mut cum_cap = 0; + let mut npart_per_dir = vec![0; data_dirs.len()]; + for (idir, dd) in data_dirs.iter().enumerate() { + if let DataDirState::Active { capacity } = dd.state { + let begin = (cum_cap * DRIVE_NPART as u64) / total_cap; + cum_cap += capacity; + let end = (cum_cap * DRIVE_NPART as u64) / total_cap; + npart_per_dir[idir] = (end - begin) as usize; + } + } + assert_eq!(cum_cap, total_cap); + assert_eq!(npart_per_dir.iter().sum::(), DRIVE_NPART); + + // For all directories that have too many primary partitions, + // move that partition to secondary + for (idir, (parts, tgt_npart)) in dir_prim.iter_mut().zip(npart_per_dir.iter()).enumerate() + { + while parts.len() > *tgt_npart { + let part = parts.pop().unwrap(); + if !part_sec[part].contains(&(idir as Idx)) { + part_sec[part].push(idir as Idx); + } + } + } + + // Calculate the vector of primary partition dir index + let mut part_prim = vec![None; DRIVE_NPART]; + for (idir, parts) in dir_prim.iter().enumerate() { + for part in parts.iter() { + assert!(part_prim[*part].is_none()); + part_prim[*part] = Some(idir as Idx) + } + } + + // Calculate a vector of unassigned partitions + let mut unassigned = part_prim + .iter() + .enumerate() + .filter(|(_, dir)| dir.is_none()) + .map(|(ipart, _)| ipart) + .collect::>(); + + // For all directories that don't have enough primary partitions, + // add partitions from unassigned + for (idir, (parts, tgt_npart)) in dir_prim.iter_mut().zip(npart_per_dir.iter()).enumerate() + { + if parts.len() < *tgt_npart { + let required = *tgt_npart - parts.len(); + assert!(unassigned.len() >= required); + for _ in 0..required { + let new_part = unassigned.pop().unwrap(); + part_prim[new_part] = Some(idir as Idx); + part_sec[new_part].retain(|x| *x != idir as Idx); + } + } + } + + // Sanity checks + assert!(part_prim.iter().all(|x| x.is_some())); + assert!(unassigned.is_empty()); + + // Transform part_prim from vec of Option to vec of Idx + let part_prim = part_prim + .into_iter() + .map(|x| x.unwrap()) + .collect::>(); + assert!(part_prim.iter().all(|p| data_dirs + .get(*p as usize) + .and_then(|x| x.capacity()) + .unwrap_or(0) + > 0)); + + // If any of the newly added storage locations is non-empty, + // it might have been removed and added again and might contain data, + // so add it as a secondary storage location for all partitions + // to make sure existing files are not lost + for (i, dd) in data_dirs.iter().enumerate() { + if self.data_dirs.iter().any(|ed| ed.path == dd.path) { + continue; + } + if dir_not_empty(&dd.path)? { + for (sec, prim) in part_sec.iter_mut().zip(part_prim.iter()) { + if *prim != i as Idx && !sec.contains(&(i as Idx)) { + sec.push(i as Idx); + } + } + } + } + + // Apply newly generated config + Ok(Self { + data_dirs, + markers: self.markers, + part_prim, + part_sec, + }) + } + + pub(crate) fn check_markers(&mut self) -> Result<(), Error> { + let data_dirs = &self.data_dirs; + self.markers + .retain(|k, _| data_dirs.iter().any(|x| x.path == *k)); + + for dir in self.data_dirs.iter() { + let mut marker_path = dir.path.clone(); + marker_path.push(MARKER_FILE_NAME); + let existing_marker = std::fs::read_to_string(&marker_path).ok(); + match (existing_marker, self.markers.get(&dir.path)) { + (Some(m1), Some(m2)) => { + if m1 != *m2 { + return Err(Error::Message(format!("Mismatched content for marker file `{}` in data directory `{}`. If you moved data directories or changed their mountpoints, you should remove the `data_layout` file in Garage's metadata directory and restart Garage.", MARKER_FILE_NAME, dir.path.display()))); + } + } + (None, Some(_)) => { + return Err(Error::Message(format!("Could not find expected marker file `{}` in data directory `{}`, make sure this data directory is mounted correctly.", MARKER_FILE_NAME, dir.path.display()))); + } + (Some(mkr), None) => { + self.markers.insert(dir.path.clone(), mkr); + } + (None, None) => { + let mkr = hex::encode(garage_util::data::gen_uuid().as_slice()); + std::fs::write(&marker_path, &mkr)?; + self.markers.insert(dir.path.clone(), mkr); + } + } + } + + Ok(()) + } + + pub(crate) fn primary_block_dir(&self, hash: &Hash) -> PathBuf { + let ipart = self.partition_from(hash); + let idir = self.part_prim[ipart] as usize; + self.block_dir_from(hash, &self.data_dirs[idir].path) + } + + pub(crate) fn secondary_block_dirs<'a>( + &'a self, + hash: &'a Hash, + ) -> impl Iterator + 'a { + let ipart = self.partition_from(hash); + self.part_sec[ipart] + .iter() + .map(move |idir| self.block_dir_from(hash, &self.data_dirs[*idir as usize].path)) + } + + fn partition_from(&self, hash: &Hash) -> usize { + u16::from_be_bytes([ + hash.as_slice()[HASH_DRIVE_BYTES.0], + hash.as_slice()[HASH_DRIVE_BYTES.1], + ]) as usize + % DRIVE_NPART + } + + fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf { + let mut path = dir.clone(); + path.push(hex::encode(&hash.as_slice()[0..1])); + path.push(hex::encode(&hash.as_slice()[1..2])); + path + } + + pub(crate) fn without_secondary_locations(&self) -> Self { + Self { + data_dirs: self.data_dirs.clone(), + markers: self.markers.clone(), + part_prim: self.part_prim.clone(), + part_sec: self.part_sec.iter().map(|_| vec![]).collect::>(), + } + } +} + +impl InitialFormat for DataLayout { + const VERSION_MARKER: &'static [u8] = b"G09bmdl"; +} + +impl DataDir { + pub fn capacity(&self) -> Option { + match self.state { + DataDirState::Active { capacity } => Some(capacity), + _ => None, + } + } +} + +fn make_data_dirs(dirs: &DataDirEnum) -> Result, Error> { + let mut data_dirs = vec![]; + match dirs { + DataDirEnum::Single(path) => data_dirs.push(DataDir { + path: path.clone(), + state: DataDirState::Active { + capacity: 1_000_000_000, // whatever, doesn't matter + }, + }), + DataDirEnum::Multiple(dirs) => { + let mut ok = false; + for dir in dirs.iter() { + let state = match &dir.capacity { + Some(cap) if dir.read_only == false => { + let capacity = cap.parse::() + .ok_or_message("invalid capacity value")?.as_u64(); + if capacity == 0 { + return Err(Error::Message(format!("data directory {} should have non-zero capacity", dir.path.to_string_lossy()))); + } + ok = true; + DataDirState::Active { + capacity, + } + } + None if dir.read_only == true => { + DataDirState::ReadOnly + } + _ => return Err(Error::Message(format!("data directories in data_dir should have a capacity value or be marked read_only, not the case for {}", dir.path.to_string_lossy()))), + }; + data_dirs.push(DataDir { + path: dir.path.clone(), + state, + }); + } + if !ok { + return Err(Error::Message( + "incorrect data_dir configuration, no primary writable directory specified" + .into(), + )); + } + } + } + Ok(data_dirs) +} + +fn dir_not_empty(path: &PathBuf) -> Result { + for entry in std::fs::read_dir(&path)? { + let dir = entry?; + let ft = dir.file_type()?; + let name = dir.file_name().into_string().ok(); + if ft.is_file() && name.as_deref() == Some(MARKER_FILE_NAME) { + return Ok(true); + } + if ft.is_dir() && name.and_then(|hex| hex::decode(&hex).ok()).is_some() { + return Ok(true); + } + } + Ok(false) +} diff --git a/src/block/lib.rs b/src/block/lib.rs index d2814f77..944f0d83 100644 --- a/src/block/lib.rs +++ b/src/block/lib.rs @@ -6,5 +6,9 @@ pub mod repair; pub mod resync; mod block; +mod layout; mod metrics; mod rc; + +pub use block::zstd_encode; +pub use rc::CalculateRefcount; diff --git a/src/block/manager.rs b/src/block/manager.rs index 26278974..96ca9c90 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -1,34 +1,32 @@ +use std::convert::TryInto; use std::path::PathBuf; -use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use arc_swap::ArcSwapOption; -use async_trait::async_trait; +use arc_swap::{ArcSwap, ArcSwapOption}; use bytes::Bytes; use rand::prelude::*; use serde::{Deserialize, Serialize}; -use futures::Stream; -use futures_util::stream::StreamExt; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::sync::{mpsc, Mutex, MutexGuard}; +use tokio::sync::{mpsc, Mutex, MutexGuard, Semaphore}; use opentelemetry::{ trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, Context, }; -use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream}; +use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream}; use garage_db as db; use garage_util::background::{vars, BackgroundRunner}; +use garage_util::config::Config; use garage_util::data::*; use garage_util::error::*; use garage_util::metrics::RecordDuration; -use garage_util::persister::PersisterShared; +use garage_util::persister::{Persister, PersisterShared}; use garage_util::time::msec_to_rfc3339; use garage_rpc::rpc_helper::OrderTag; @@ -38,6 +36,7 @@ use garage_rpc::*; use garage_table::replication::{TableReplication, TableShardedReplication}; use crate::block::*; +use crate::layout::*; use crate::metrics::*; use crate::rc::*; use crate::repair::*; @@ -51,6 +50,8 @@ pub const INLINE_THRESHOLD: usize = 3072; // to delete the block locally. pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600); +const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15); + /// RPC messages used to share blocks of data between nodes #[derive(Debug, Serialize, Deserialize)] pub enum BlockRpc { @@ -77,18 +78,25 @@ impl Rpc for BlockRpc { pub struct BlockManager { /// Replication strategy, allowing to find on which node blocks should be located pub replication: TableShardedReplication, - /// Directory in which block are stored - pub data_dir: PathBuf, + /// Data layout + pub(crate) data_layout: ArcSwap, + /// Data layout persister + pub(crate) data_layout_persister: Persister, + + data_fsync: bool, compression_level: Option, + disable_scrub: bool, - mutation_lock: [Mutex; 256], + mutation_lock: Vec>, + read_semaphore: Semaphore, - pub(crate) rc: BlockRc, + pub rc: BlockRc, pub resync: BlockResyncManager, pub(crate) system: Arc, pub(crate) endpoint: Arc>, + buffer_kb_semaphore: Arc, pub(crate) metrics: BlockManagerMetrics, @@ -105,6 +113,9 @@ pub struct BlockResyncErrorInfo { pub next_try: u64, } +// The number of different mutexes used to parallelize write access to data blocks +const MUTEX_COUNT: usize = 256; + // This custom struct contains functions that must only be ran // when the lock is held. We ensure that it is the case by storing // it INSIDE a Mutex. @@ -113,11 +124,27 @@ struct BlockManagerLocked(); impl BlockManager { pub fn new( db: &db::Db, - data_dir: PathBuf, - compression_level: Option, + config: &Config, replication: TableShardedReplication, system: Arc, - ) -> Arc { + ) -> Result, Error> { + // Load or compute layout, i.e. assignment of data blocks to the different data directories + let data_layout_persister: Persister = + Persister::new(&system.metadata_dir, "data_layout"); + let mut data_layout = match data_layout_persister.load() { + Ok(layout) => layout + .update(&config.data_dir) + .ok_or_message("invalid data_dir config")?, + Err(_) => { + DataLayout::initialize(&config.data_dir).ok_or_message("invalid data_dir config")? + } + }; + data_layout.check_markers()?; + data_layout_persister + .save(&data_layout) + .expect("cannot save data_layout"); + + // Open metadata tables let rc = db .open_tree("block_local_rc") .expect("Unable to open block_local_rc tree"); @@ -129,24 +156,36 @@ impl BlockManager { .netapp .endpoint("garage_block/manager.rs/Rpc".to_string()); + let buffer_kb_semaphore = Arc::new(Semaphore::new(config.block_ram_buffer_max / 1024)); + let metrics = BlockManagerMetrics::new( - compression_level, - rc.rc.clone(), + config.compression_level, + rc.rc_table.clone(), resync.queue.clone(), resync.errors.clone(), + buffer_kb_semaphore.clone(), ); let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info"); let block_manager = Arc::new(Self { replication, - data_dir, - compression_level, - mutation_lock: [(); 256].map(|_| Mutex::new(BlockManagerLocked())), + data_layout: ArcSwap::new(Arc::new(data_layout)), + data_layout_persister, + data_fsync: config.data_fsync, + disable_scrub: config.disable_scrub, + compression_level: config.compression_level, + mutation_lock: vec![(); MUTEX_COUNT] + .iter() + .map(|_| Mutex::new(BlockManagerLocked())) + .collect::>(), + + read_semaphore: Semaphore::new(config.block_max_concurrent_reads), rc, resync, system, endpoint, + buffer_kb_semaphore, metrics, scrub_persister, tx_scrub_command: ArcSwapOption::new(None), @@ -154,7 +193,7 @@ impl BlockManager { block_manager.endpoint.set_handler(block_manager.clone()); block_manager.scrub_persister.set_with(|_| ()).unwrap(); - block_manager + Ok(block_manager) } pub fn spawn_workers(self: &Arc, bg: &BackgroundRunner) { @@ -165,33 +204,43 @@ impl BlockManager { } // Spawn scrub worker - let (scrub_tx, scrub_rx) = mpsc::channel(1); - self.tx_scrub_command.store(Some(Arc::new(scrub_tx))); - bg.spawn_worker(ScrubWorker::new( - self.clone(), - scrub_rx, - self.scrub_persister.clone(), - )); + if !self.disable_scrub { + let (scrub_tx, scrub_rx) = mpsc::channel(1); + self.tx_scrub_command.store(Some(Arc::new(scrub_tx))); + bg.spawn_worker(ScrubWorker::new( + self.clone(), + scrub_rx, + self.scrub_persister.clone(), + )); + } } pub fn register_bg_vars(&self, vars: &mut vars::BgVars) { self.resync.register_bg_vars(vars); - vars.register_rw( - &self.scrub_persister, - "scrub-tranquility", - |p| p.get_with(|x| x.tranquility), - |p, tranquility| p.set_with(|x| x.tranquility = tranquility), - ); - vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| { - p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub)) - }); - vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| { - p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub)) - }); - vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| { - p.get_with(|x| x.corruptions_detected) - }); + if !self.disable_scrub { + vars.register_rw( + &self.scrub_persister, + "scrub-tranquility", + |p| p.get_with(|x| x.tranquility), + |p, tranquility| p.set_with(|x| x.tranquility = tranquility), + ); + vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| { + p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub)) + }); + vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| { + p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub)) + }); + vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| { + p.get_with(|x| x.corruptions_detected) + }); + } + } + + /// Initialization: set how block references are recalculated + /// for repair operations + pub fn set_recalc_rc(&self, recalc: Vec) { + self.rc.recalc_rc.store(Some(Arc::new(recalc))); } /// Ask nodes that might have a (possibly compressed) block for it @@ -199,46 +248,16 @@ impl BlockManager { async fn rpc_get_raw_block_streaming( &self, hash: &Hash, + priority: RequestPriority, order_tag: Option, - ) -> Result<(DataBlockHeader, ByteStream), Error> { - let who = self.replication.read_nodes(hash); - let who = self.system.rpc.request_order(&who); - - for node in who.iter() { - let node_id = NodeID::from(*node); - let rpc = self.endpoint.call_streaming( - &node_id, - BlockRpc::GetBlock(*hash, order_tag), - PRIO_NORMAL | PRIO_SECONDARY, - ); - tokio::select! { - res = rpc => { - let res = match res { - Ok(res) => res, - Err(e) => { - debug!("Node {:?} returned error: {}", node, e); - continue; - } - }; - let (header, stream) = match res.into_parts() { - (Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream), - _ => { - debug!("Node {:?} returned a malformed response", node); - continue; - } - }; - return Ok((header, stream)); - } - _ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => { - debug!("Node {:?} didn't return block in time, trying next.", node); - } - }; - } - - Err(Error::Message(format!( - "Unable to read block {:?}: no node returned a valid block", - hash - ))) + ) -> Result { + self.rpc_get_raw_block_internal( + hash, + priority, + order_tag, + |stream| async move { Ok(stream) }, + ) + .await } /// Ask nodes that might have a (possibly compressed) block for it @@ -246,66 +265,96 @@ impl BlockManager { pub(crate) async fn rpc_get_raw_block( &self, hash: &Hash, + priority: RequestPriority, order_tag: Option, ) -> Result { - let who = self.replication.read_nodes(hash); - let who = self.system.rpc.request_order(&who); + self.rpc_get_raw_block_internal(hash, priority, order_tag, |block_stream| async move { + let (header, stream) = block_stream.into_parts(); + read_stream_to_end(stream) + .await + .err_context("error in block data stream") + .map(|data| DataBlock::from_parts(header, data.into_bytes())) + }) + .await + } + + async fn rpc_get_raw_block_internal( + &self, + hash: &Hash, + priority: RequestPriority, + order_tag: Option, + f: F, + ) -> Result + where + F: Fn(DataBlockStream) -> Fut, + Fut: futures::Future>, + { + let who = self + .system + .rpc_helper() + .block_read_nodes_of(hash, self.system.rpc_helper()); for node in who.iter() { let node_id = NodeID::from(*node); let rpc = self.endpoint.call_streaming( &node_id, BlockRpc::GetBlock(*hash, order_tag), - PRIO_NORMAL | PRIO_SECONDARY, + priority, ); tokio::select! { res = rpc => { let res = match res { Ok(res) => res, Err(e) => { - debug!("Node {:?} returned error: {}", node, e); + debug!("Get block {:?}: node {:?} could not be contacted: {}", hash, node, e); continue; } }; - let (header, stream) = match res.into_parts() { - (Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream), - _ => { - debug!("Node {:?} returned a malformed response", node); + let block_stream = match res.into_parts() { + (Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => DataBlockStream::from_parts(header, stream), + (Ok(_), _) => { + debug!("Get block {:?}: node {:?} returned a malformed response", hash, node); + continue; + } + (Err(e), _) => { + debug!("Get block {:?}: node {:?} returned error: {}", hash, node, e); continue; } }; - match read_stream_to_end(stream).await { - Ok(bytes) => return Ok(DataBlock::from_parts(header, bytes)), + match f(block_stream).await { + Ok(ret) => return Ok(ret), Err(e) => { - debug!("Error reading stream from node {:?}: {}", node, e); + debug!("Get block {:?}: error reading stream from node {:?}: {}", hash, node, e); } } } - _ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => { - debug!("Node {:?} didn't return block in time, trying next.", node); + // TODO: sleep less long (fail early), initiate a second request earlier + // if the first one doesn't succeed rapidly + // TODO: keep first request running when initiating a new one and take the + // one that finishes earlier + _ = tokio::time::sleep(self.system.rpc_helper().rpc_timeout()) => { + debug!("Get block {:?}: node {:?} didn't return block in time, trying next.", hash, node); } }; } - Err(Error::Message(format!( - "Unable to read block {:?}: no node returned a valid block", - hash - ))) + let err = Error::MissingBlock(*hash); + debug!("{}", err); + Err(err) } // ---- Public interface ---- - /// Ask nodes that might have a block for it, - /// return it as a stream + /// Ask nodes that might have a block for it, return it as a stream pub async fn rpc_get_block_streaming( &self, hash: &Hash, order_tag: Option, - ) -> Result< - Pin> + Send + Sync + 'static>>, - Error, - > { - let (header, stream) = self.rpc_get_raw_block_streaming(hash, order_tag).await?; + ) -> Result { + let block_stream = self + .rpc_get_raw_block_streaming(hash, PRIO_NORMAL | PRIO_SECONDARY, order_tag) + .await?; + let (header, stream) = block_stream.into_parts(); match header { DataBlockHeader::Plain => Ok(stream), DataBlockHeader::Compressed => { @@ -318,34 +367,44 @@ impl BlockManager { } } - /// Ask nodes that might have a block for it - pub async fn rpc_get_block( - &self, - hash: &Hash, - order_tag: Option, - ) -> Result { - self.rpc_get_raw_block(hash, order_tag) - .await? - .verify_get(*hash) - } - /// Send block to nodes that should have it - pub async fn rpc_put_block(&self, hash: Hash, data: Bytes) -> Result<(), Error> { - let who = self.replication.write_nodes(&hash); + pub async fn rpc_put_block( + &self, + hash: Hash, + data: Bytes, + prevent_compression: bool, + order_tag: Option, + ) -> Result<(), Error> { + let who = self.system.cluster_layout().current_storage_nodes_of(&hash); - let (header, bytes) = DataBlock::from_buffer(data, self.compression_level) + let compression_level = self.compression_level.filter(|_| !prevent_compression); + let (header, bytes) = DataBlock::from_buffer(data, compression_level) .await .into_parts(); + + let permit = self + .buffer_kb_semaphore + .clone() + .acquire_many_owned((bytes.len() / 1024).try_into().unwrap()) + .await + .ok_or_message("could not reserve space for buffer of data to send to remote nodes")?; + let put_block_rpc = Req::new(BlockRpc::PutBlock { hash, header })?.with_stream_from_buffer(bytes); + let put_block_rpc = if let Some(tag) = order_tag { + put_block_rpc.with_order_tag(tag) + } else { + put_block_rpc + }; self.system - .rpc - .try_call_many( + .rpc_helper() + .try_write_many_sets( &self.endpoint, - &who[..], + &[who], put_block_rpc, RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY) + .with_drop_on_completion(permit) .with_quorum(self.replication.write_quorum()), ) .await?; @@ -354,13 +413,8 @@ impl BlockManager { } /// Get number of items in the refcount table - pub fn rc_len(&self) -> Result { - Ok(self.rc.rc.len()?) - } - - /// Get number of items in the refcount table - pub fn rc_fast_len(&self) -> Result, Error> { - Ok(self.rc.rc.fast_len()?) + pub fn rc_approximate_len(&self) -> Result { + Ok(self.rc.rc_table.approximate_len()?) } /// Send command to start/stop/manager scrub worker @@ -378,7 +432,7 @@ impl BlockManager { /// List all resync errors pub fn list_resync_errors(&self) -> Result, Error> { - let mut blocks = Vec::with_capacity(self.resync.errors.len()); + let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?); for ent in self.resync.errors.iter()? { let (hash, cnt) = ent?; let cnt = ErrorCounter::decode(&cnt); @@ -416,7 +470,7 @@ impl BlockManager { tokio::spawn(async move { if let Err(e) = this .resync - .put_to_resync(&hash, 2 * this.system.rpc.rpc_timeout()) + .put_to_resync(&hash, 2 * this.system.rpc_helper().rpc_timeout()) { error!("Block {:?} could not be put in resync queue: {}.", hash, e); } @@ -459,7 +513,7 @@ impl BlockManager { stream: Option, ) -> Result<(), Error> { let stream = stream.ok_or_message("missing stream")?; - let bytes = read_stream_to_end(stream).await?; + let bytes = read_stream_to_end(stream).await?.into_bytes(); let data = DataBlock::from_parts(header, bytes); self.write_block(&hash, &data).await } @@ -468,8 +522,6 @@ impl BlockManager { pub(crate) async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<(), Error> { let tracer = opentelemetry::global::tracer("garage"); - let write_size = data.inner_buffer().len() as u64; - self.lock_mutate(hash) .await .write_block(hash, data, self) @@ -479,8 +531,6 @@ impl BlockManager { )) .await?; - self.metrics.bytes_written.add(write_size); - Ok(()) } @@ -507,70 +557,75 @@ impl BlockManager { /// Read block from disk, verifying it's integrity pub(crate) async fn read_block(&self, hash: &Hash) -> Result { - let data = self - .read_block_internal(hash) - .bound_record_duration(&self.metrics.block_read_duration) - .await?; - - self.metrics - .bytes_read - .add(data.inner_buffer().len() as u64); - - Ok(data) + let tracer = opentelemetry::global::tracer("garage"); + async { + match self.find_block(hash).await { + Some(p) => self.read_block_from(hash, &p).await, + None => { + return Err(Error::Message(format!( + "block {:?} not found on node", + hash + ))); + } + } + } + .bound_record_duration(&self.metrics.block_read_duration) + .with_context(Context::current_with_span( + tracer.start("BlockManager::read_block"), + )) + .await } - async fn read_block_internal(&self, hash: &Hash) -> Result { - let mut path = self.block_path(hash); - let compressed = match self.is_block_compressed(hash).await { - Ok(c) => c, - Err(e) => { - // Not found but maybe we should have had it ?? - self.resync - .put_to_resync(hash, 2 * self.system.rpc.rpc_timeout())?; - return Err(Into::into(e)); + pub(crate) async fn read_block_from( + &self, + hash: &Hash, + block_path: &DataBlockPath, + ) -> Result { + let (header, path) = block_path.as_parts_ref(); + + let permit = tokio::select! { + sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?, + _ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => { + self.metrics.block_read_semaphore_timeouts.add(1); + debug!("read block {:?}: read_semaphore acquire timeout", hash); + return Err(Error::Message("read block: read_semaphore acquire timeout".into())); } }; - if compressed { - path.set_extension("zst"); - } - let mut f = fs::File::open(&path).await?; + let mut f = fs::File::open(&path).await?; let mut data = vec![]; f.read_to_end(&mut data).await?; + self.metrics.bytes_read.add(data.len() as u64); drop(f); - let data = if compressed { - DataBlock::Compressed(data.into()) - } else { - DataBlock::Plain(data.into()) - }; + let data = DataBlock::from_parts(header, data.into()); if data.verify(*hash).is_err() { self.metrics.corruption_counter.add(1); + warn!( + "Block {:?} is corrupted. Renaming to .corrupted and resyncing.", + hash + ); self.lock_mutate(hash) .await - .move_block_to_corrupted(hash, self) + .move_block_to_corrupted(block_path) .await?; self.resync.put_to_resync(hash, Duration::from_millis(0))?; + return Err(Error::CorruptData(*hash)); } + drop(permit); + Ok(data) } - /// Check if this node has a block and whether it needs it - pub(crate) async fn check_block_status(&self, hash: &Hash) -> Result { - self.lock_mutate(hash) - .await - .check_block_status(hash, self) - .await - } - /// Check if this node should have a block, but don't actually have it async fn need_block(&self, hash: &Hash) -> Result { - let BlockStatus { exists, needed } = self.check_block_status(hash).await?; - Ok(needed.is_nonzero() && !exists) + let rc = self.rc.get_block_rc(hash)?; + let exists = self.find_block(hash).await.is_some(); + Ok(rc.is_nonzero() && !exists) } /// Delete block if it is not needed anymore @@ -581,52 +636,79 @@ impl BlockManager { .await } - /// Utility: gives the path of the directory in which a block should be found - fn block_dir(&self, hash: &Hash) -> PathBuf { - let mut path = self.data_dir.clone(); - path.push(hex::encode(&hash.as_slice()[0..1])); - path.push(hex::encode(&hash.as_slice()[1..2])); - path - } + /// Find the path where a block is currently stored + pub(crate) async fn find_block(&self, hash: &Hash) -> Option { + let data_layout = self.data_layout.load_full(); + let dirs = Some(data_layout.primary_block_dir(hash)) + .into_iter() + .chain(data_layout.secondary_block_dirs(hash)); + let filename = hex::encode(hash.as_ref()); - /// Utility: give the full path where a block should be found, minus extension if block is - /// compressed - fn block_path(&self, hash: &Hash) -> PathBuf { - let mut path = self.block_dir(hash); - path.push(hex::encode(hash.as_ref())); - path - } + for dir in dirs { + let mut path = dir; + path.push(&filename); - /// Utility: check if block is stored compressed. Error if block is not stored - async fn is_block_compressed(&self, hash: &Hash) -> Result { - let mut path = self.block_path(hash); - path.set_extension("zst"); - if fs::metadata(&path).await.is_ok() { - return Ok(true); + if self.compression_level.is_none() { + // If compression is disabled on node - check for the raw block + // first and then a compressed one (as compression may have been + // previously enabled). + if fs::metadata(&path).await.is_ok() { + return Some(DataBlockPath::plain(path)); + } + path.set_extension("zst"); + if fs::metadata(&path).await.is_ok() { + return Some(DataBlockPath::compressed(path)); + } + } else { + path.set_extension("zst"); + if fs::metadata(&path).await.is_ok() { + return Some(DataBlockPath::compressed(path)); + } + path.set_extension(""); + if fs::metadata(&path).await.is_ok() { + return Some(DataBlockPath::plain(path)); + } + } } - path.set_extension(""); - fs::metadata(&path).await.map(|_| false).map_err(Into::into) + + None + } + + /// Rewrite a block at the primary location for its path and delete the old path. + /// Returns the number of bytes read/written + pub(crate) async fn fix_block_location( + &self, + hash: &Hash, + wrong_path: DataBlockPath, + ) -> Result { + let data = self.read_block_from(hash, &wrong_path).await?; + self.lock_mutate(hash) + .await + .write_block_inner(hash, &data, self, Some(wrong_path)) + .await?; + Ok(data.as_parts_ref().1.len()) } async fn lock_mutate(&self, hash: &Hash) -> MutexGuard<'_, BlockManagerLocked> { let tracer = opentelemetry::global::tracer("garage"); - self.mutation_lock[hash.as_slice()[0] as usize] + let ilock = u16::from_be_bytes([hash.as_slice()[0], hash.as_slice()[1]]) as usize + % self.mutation_lock.len(); + self.mutation_lock[ilock] .lock() .with_context(Context::current_with_span( - tracer.start("Acquire mutation_lock"), + tracer.start(format!("Acquire mutation_lock #{}", ilock)), )) .await } } -#[async_trait] impl StreamingEndpointHandler for BlockManager { async fn handle(self: &Arc, mut message: Req, _from: NodeID) -> Resp { match message.msg() { BlockRpc::PutBlock { hash, header } => Resp::new( self.handle_put_block(*hash, *header, message.take_stream()) .await - .map(|_| BlockRpc::Ok), + .map(|()| BlockRpc::Ok), ), BlockRpc::GetBlock(h, order_tag) => self.handle_get_block(h, *order_tag).await, BlockRpc::NeedBlockQuery(h) => { @@ -637,66 +719,80 @@ impl StreamingEndpointHandler for BlockManager { } } -pub(crate) struct BlockStatus { - pub(crate) exists: bool, - pub(crate) needed: RcEntry, -} - impl BlockManagerLocked { - async fn check_block_status( - &self, - hash: &Hash, - mgr: &BlockManager, - ) -> Result { - let exists = mgr.is_block_compressed(hash).await.is_ok(); - let needed = mgr.rc.get_block_rc(hash)?; - - Ok(BlockStatus { exists, needed }) - } - async fn write_block( &self, hash: &Hash, data: &DataBlock, mgr: &BlockManager, ) -> Result<(), Error> { - let compressed = data.is_compressed(); - let data = data.inner_buffer(); + let existing_path = mgr.find_block(hash).await; + self.write_block_inner(hash, data, mgr, existing_path).await + } - let mut path = mgr.block_dir(hash); - let directory = path.clone(); - path.push(hex::encode(hash)); + async fn write_block_inner( + &self, + hash: &Hash, + data: &DataBlock, + mgr: &BlockManager, + existing_path: Option, + ) -> Result<(), Error> { + let (header, data) = data.as_parts_ref(); + let compressed = header.is_compressed(); - fs::create_dir_all(&directory).await?; + let directory = mgr.data_layout.load().primary_block_dir(hash); - let to_delete = match (mgr.is_block_compressed(hash).await, compressed) { - (Ok(true), _) => return Ok(()), - (Ok(false), false) => return Ok(()), - (Ok(false), true) => { - let path_to_delete = path.clone(); - path.set_extension("zst"); - Some(path_to_delete) - } - (Err(_), compressed) => { - if compressed { - path.set_extension("zst"); - } - None - } + let mut tgt_path = directory.clone(); + tgt_path.push(hex::encode(hash)); + if compressed { + tgt_path.set_extension("zst"); + } + + let existing_info = existing_path.map(|x| x.into_parts()); + let to_delete = match (existing_info, compressed) { + // If the block is stored in the wrong directory, + // write it again at the correct path and delete the old path + (Some((DataBlockHeader::Plain, p)), false) if p != tgt_path => Some(p), + (Some((DataBlockHeader::Compressed, p)), true) if p != tgt_path => Some(p), + + // If the block is already stored not compressed but we have a compressed + // copy, write the compressed copy and delete the uncompressed one + (Some((DataBlockHeader::Plain, plain_path)), true) => Some(plain_path), + + // If the block is already stored compressed, + // keep the stored copy, we have nothing to do + (Some((DataBlockHeader::Compressed, _)), _) => return Ok(()), + + // If the block is already stored not compressed, + // and we don't have a compressed copy either, + // keep the stored copy, we have nothing to do + (Some((DataBlockHeader::Plain, _)), false) => return Ok(()), + + // If the block isn't stored already, just store what is given to us + (None, _) => None, }; + assert!(to_delete.as_ref() != Some(&tgt_path)); - let mut path_tmp = path.clone(); + let mut path_tmp = tgt_path.clone(); let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>())); path_tmp.set_extension(tmp_extension); + fs::create_dir_all(&directory).await?; + let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone())); let mut f = fs::File::create(&path_tmp).await?; f.write_all(data).await?; - f.sync_all().await?; + f.flush().await?; + mgr.metrics.bytes_written.add(data.len() as u64); + + if mgr.data_fsync { + f.sync_all().await?; + } + drop(f); - fs::rename(path_tmp, path).await?; + fs::rename(path_tmp, tgt_path).await?; delete_on_drop.cancel(); @@ -704,68 +800,51 @@ impl BlockManagerLocked { fs::remove_file(to_delete).await?; } - // We want to ensure that when this function returns, data is properly persisted - // to disk. The first step is the sync_all above that does an fsync on the data file. - // Now, we do an fsync on the containing directory, to ensure that the rename - // is persisted properly. See: - // http://thedjbway.b0llix.net/qmail/syncdir.html - let dir = fs::OpenOptions::new() - .read(true) - .mode(0) - .open(directory) - .await?; - dir.sync_all().await?; - drop(dir); + if mgr.data_fsync { + // We want to ensure that when this function returns, data is properly persisted + // to disk. The first step is the sync_all above that does an fsync on the data file. + // Now, we do an fsync on the containing directory, to ensure that the rename + // is persisted properly. See: + // http://thedjbway.b0llix.net/qmail/syncdir.html + let dir = fs::OpenOptions::new() + .read(true) + .mode(0) + .open(directory) + .await?; + dir.sync_all().await?; + drop(dir); + } Ok(()) } - async fn move_block_to_corrupted(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> { - warn!( - "Block {:?} is corrupted. Renaming to .corrupted and resyncing.", - hash - ); - let mut path = mgr.block_path(hash); + async fn move_block_to_corrupted(&self, block_path: &DataBlockPath) -> Result<(), Error> { + let (header, path) = block_path.as_parts_ref(); + let mut path2 = path.clone(); - if mgr.is_block_compressed(hash).await? { - path.set_extension("zst"); + if header.is_compressed() { path2.set_extension("zst.corrupted"); } else { path2.set_extension("corrupted"); } + fs::rename(path, path2).await?; Ok(()) } async fn delete_if_unneeded(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> { - let BlockStatus { exists, needed } = self.check_block_status(hash, mgr).await?; - - if exists && needed.is_deletable() { - let mut path = mgr.block_path(hash); - if mgr.is_block_compressed(hash).await? { - path.set_extension("zst"); + let rc = mgr.rc.get_block_rc(hash)?; + if rc.is_deletable() { + while let Some(path) = mgr.find_block(hash).await { + let (_header, path) = path.as_parts_ref(); + fs::remove_file(path).await?; + mgr.metrics.delete_counter.add(1); } - fs::remove_file(path).await?; - mgr.metrics.delete_counter.add(1); } Ok(()) } } -async fn read_stream_to_end(mut stream: ByteStream) -> Result { - let mut parts: Vec = vec![]; - while let Some(part) = stream.next().await { - parts.push(part.ok_or_message("error in stream")?); - } - - Ok(parts - .iter() - .map(|x| &x[..]) - .collect::>() - .concat() - .into()) -} - struct DeleteOnDrop(Option); impl DeleteOnDrop { diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 6659df32..81021fe1 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,7 +1,10 @@ +use std::sync::Arc; + +use tokio::sync::Semaphore; + use opentelemetry::{global, metrics::*}; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { @@ -9,6 +12,7 @@ pub struct BlockManagerMetrics { pub(crate) _rc_size: ValueObserver, pub(crate) _resync_queue_len: ValueObserver, pub(crate) _resync_errored_blocks: ValueObserver, + pub(crate) _buffer_free_kb: ValueObserver, pub(crate) resync_counter: BoundCounter, pub(crate) resync_error_counter: BoundCounter, @@ -18,6 +22,7 @@ pub struct BlockManagerMetrics { pub(crate) bytes_read: BoundCounter, pub(crate) block_read_duration: BoundValueRecorder, + pub(crate) block_read_semaphore_timeouts: BoundCounter, pub(crate) bytes_written: BoundCounter, pub(crate) block_write_duration: BoundValueRecorder, pub(crate) delete_counter: BoundCounter, @@ -29,8 +34,9 @@ impl BlockManagerMetrics { pub fn new( compression_level: Option, rc_tree: db::Tree, - resync_queue: CountedTree, - resync_errors: CountedTree, + resync_queue: db::Tree, + resync_errors: db::Tree, + buffer_semaphore: Arc, ) -> Self { let meter = global::meter("garage_model/block"); Self { @@ -45,15 +51,17 @@ impl BlockManagerMetrics { .init(), _rc_size: meter .u64_value_observer("block.rc_size", move |observer| { - if let Ok(Some(v)) = rc_tree.fast_len() { - observer.observe(v as u64, &[]) + if let Ok(value) = rc_tree.approximate_len() { + observer.observe(value as u64, &[]) } }) .with_description("Number of blocks known to the reference counter") .init(), _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - observer.observe(resync_queue.len() as u64, &[]) + if let Ok(value) = resync_queue.approximate_len() { + observer.observe(value as u64, &[]); + } }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -61,11 +69,22 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - observer.observe(resync_errors.len() as u64, &[]) + if let Ok(value) = resync_errors.approximate_len() { + observer.observe(value as u64, &[]); + } }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), + _buffer_free_kb: meter + .u64_value_observer("block.ram_buffer_free_kb", move |observer| { + observer.observe(buffer_semaphore.available_permits() as u64, &[]) + }) + .with_description( + "Available RAM in KiB to use for buffering data blocks to be written to remote nodes", + ) + .init(), + resync_counter: meter .u64_counter("block.resync_counter") .with_description("Number of calls to resync_block") @@ -101,6 +120,11 @@ impl BlockManagerMetrics { .with_description("Duration of block read operations") .init() .bind(&[]), + block_read_semaphore_timeouts: meter + .u64_counter("block.read_semaphore_timeouts") + .with_description("Number of block reads that failed due to semaphore acquire timeout") + .init() + .bind(&[]), bytes_written: meter .u64_counter("block.bytes_written") .with_description("Number of bytes written to disk") diff --git a/src/block/rc.rs b/src/block/rc.rs index 94cb5eea..4a55ee29 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -1,5 +1,7 @@ use std::convert::TryInto; +use arc_swap::ArcSwapOption; + use garage_db as db; use garage_util::data::*; @@ -8,13 +10,20 @@ use garage_util::time::*; use crate::manager::BLOCK_GC_DELAY; +pub type CalculateRefcount = + Box db::TxResult + Send + Sync>; + pub struct BlockRc { - pub(crate) rc: db::Tree, + pub rc_table: db::Tree, + pub(crate) recalc_rc: ArcSwapOption>, } impl BlockRc { pub(crate) fn new(rc: db::Tree) -> Self { - Self { rc } + Self { + rc_table: rc, + recalc_rc: ArcSwapOption::new(None), + } } /// Increment the reference counter associated to a hash. @@ -24,9 +33,9 @@ impl BlockRc { tx: &mut db::Transaction, hash: &Hash, ) -> db::TxOpResult { - let old_rc = RcEntry::parse_opt(tx.get(&self.rc, hash)?); + let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); match old_rc.increment().serialize() { - Some(x) => tx.insert(&self.rc, hash, x)?, + Some(x) => tx.insert(&self.rc_table, hash, x)?, None => unreachable!(), }; Ok(old_rc.is_zero()) @@ -39,35 +48,87 @@ impl BlockRc { tx: &mut db::Transaction, hash: &Hash, ) -> db::TxOpResult { - let new_rc = RcEntry::parse_opt(tx.get(&self.rc, hash)?).decrement(); + let new_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?).decrement(); match new_rc.serialize() { - Some(x) => tx.insert(&self.rc, hash, x)?, - None => tx.remove(&self.rc, hash)?, + Some(x) => tx.insert(&self.rc_table, hash, x)?, + None => tx.remove(&self.rc_table, hash)?, }; Ok(matches!(new_rc, RcEntry::Deletable { .. })) } /// Read a block's reference count pub(crate) fn get_block_rc(&self, hash: &Hash) -> Result { - Ok(RcEntry::parse_opt(self.rc.get(hash.as_ref())?)) + Ok(RcEntry::parse_opt(self.rc_table.get(hash.as_ref())?)) } /// Delete an entry in the RC table if it is deletable and the /// deletion time has passed pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> { let now = now_msec(); - self.rc.db().transaction(|mut tx| { - let rcval = RcEntry::parse_opt(tx.get(&self.rc, hash)?); + self.rc_table.db().transaction(|tx| { + let rcval = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); match rcval { RcEntry::Deletable { at_time } if now > at_time => { - tx.remove(&self.rc, hash)?; + tx.remove(&self.rc_table, hash)?; } _ => (), }; - tx.commit(()) + Ok(()) })?; Ok(()) } + + /// Recalculate the reference counter of a block + /// to fix potential inconsistencies + pub fn recalculate_rc(&self, hash: &Hash) -> Result<(usize, bool), Error> { + if let Some(recalc_fns) = self.recalc_rc.load().as_ref() { + trace!("Repair block RC for {:?}", hash); + let res = self + .rc_table + .db() + .transaction(|tx| { + let mut cnt = 0; + for f in recalc_fns.iter() { + cnt += f(&tx, hash)?; + } + let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?); + trace!( + "Block RC for {:?}: stored={}, calculated={}", + hash, + old_rc.as_u64(), + cnt + ); + if cnt as u64 != old_rc.as_u64() { + warn!( + "Fixing inconsistent block RC for {:?}: was {}, should be {}", + hash, + old_rc.as_u64(), + cnt + ); + let new_rc = if cnt > 0 { + RcEntry::Present { count: cnt as u64 } + } else { + RcEntry::Deletable { + at_time: now_msec() + BLOCK_GC_DELAY.as_millis() as u64, + } + }; + tx.insert(&self.rc_table, hash, new_rc.serialize().unwrap())?; + Ok((cnt, true)) + } else { + Ok((cnt, false)) + } + }) + .map_err(Error::from); + if let Err(e) = &res { + error!("Failed to fix RC for block {:?}: {}", hash, e); + } + res + } else { + Err(Error::Message( + "Block RC recalculation is not available at this point".into(), + )) + } + } } /// Describes the state of the reference counter for a block diff --git a/src/block/repair.rs b/src/block/repair.rs index 37ccd59a..ef271094 100644 --- a/src/block/repair.rs +++ b/src/block/repair.rs @@ -17,6 +17,7 @@ use garage_util::persister::PersisterShared; use garage_util::time::*; use garage_util::tranquilizer::Tranquilizer; +use crate::block::*; use crate::manager::*; // Full scrub every 25 days with a random element of 10 days mixed in below @@ -106,7 +107,7 @@ impl Worker for RepairWorker { for entry in self .manager .rc - .rc + .rc_table .range::<&[u8], _>((start_bound, Bound::Unbounded))? { let (hash, _) = entry?; @@ -136,7 +137,7 @@ impl Worker for RepairWorker { // Lists all blocks on disk and adds them to the resync queue. // This allows us to find blocks we are storing but don't actually need, // so that we can offload them if necessary and then delete them locally. - if let Some(hash) = bi.next().await? { + if let Some((_path, hash)) = bi.next().await? { self.manager .resync .put_to_resync(&hash, Duration::from_secs(0))?; @@ -175,7 +176,9 @@ mod v081 { } mod v082 { + use garage_util::data::Hash; use serde::{Deserialize, Serialize}; + use std::path::PathBuf; use super::v081; @@ -185,6 +188,27 @@ mod v082 { pub(crate) time_last_complete_scrub: u64, pub(crate) time_next_run_scrub: u64, pub(crate) corruptions_detected: u64, + #[serde(default)] + pub(crate) checkpoint: Option, + } + + #[derive(Serialize, Deserialize, Clone)] + pub struct BlockStoreIterator { + pub todo: Vec, + } + + #[derive(Serialize, Deserialize, Clone)] + pub enum BsiTodo { + Directory { + path: PathBuf, + progress_min: u64, + progress_max: u64, + }, + File { + path: PathBuf, + hash: Hash, + progress: u64, + }, } impl garage_util::migrate::Migrate for ScrubWorkerPersisted { @@ -199,6 +223,7 @@ mod v082 { time_last_complete_scrub: old.time_last_complete_scrub, time_next_run_scrub: randomize_next_scrub_run_time(old.time_last_complete_scrub), corruptions_detected: old.corruptions_detected, + checkpoint: None, } } } @@ -220,14 +245,12 @@ fn randomize_next_scrub_run_time(timestamp: u64) -> u64 { // Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to // balance scrub load across different cluster nodes. - let next_run_timestamp = timestamp + timestamp + SCRUB_INTERVAL .saturating_add(Duration::from_secs( rand::thread_rng().gen_range(0..3600 * 24 * 10), )) - .as_millis() as u64; - - next_run_timestamp + .as_millis() as u64 } impl Default for ScrubWorkerPersisted { @@ -237,22 +260,27 @@ impl Default for ScrubWorkerPersisted { time_next_run_scrub: randomize_next_scrub_run_time(now_msec()), tranquility: INITIAL_SCRUB_TRANQUILITY, corruptions_detected: 0, + checkpoint: None, } } } +#[derive(Default)] enum ScrubWorkerState { - Running(BlockStoreIterator), - Paused(BlockStoreIterator, u64), // u64 = time when to resume scrub + Running { + iterator: BlockStoreIterator, + // time of the last checkpoint + t_cp: u64, + }, + Paused { + iterator: BlockStoreIterator, + // time at which the scrub should be resumed + t_resume: u64, + }, + #[default] Finished, } -impl Default for ScrubWorkerState { - fn default() -> Self { - ScrubWorkerState::Finished - } -} - #[derive(Debug)] pub enum ScrubWorkerCommand { Start, @@ -267,10 +295,17 @@ impl ScrubWorker { rx_cmd: mpsc::Receiver, persister: PersisterShared, ) -> Self { + let work = match persister.get_with(|x| x.checkpoint.clone()) { + None => ScrubWorkerState::Finished, + Some(iterator) => ScrubWorkerState::Running { + iterator, + t_cp: now_msec(), + }, + }; Self { manager, rx_cmd, - work: ScrubWorkerState::Finished, + work, tranquilizer: Tranquilizer::new(30), persister, } @@ -281,8 +316,18 @@ impl ScrubWorker { ScrubWorkerCommand::Start => { self.work = match std::mem::take(&mut self.work) { ScrubWorkerState::Finished => { + info!("Scrub worker initializing, now performing datastore scrub"); let iterator = BlockStoreIterator::new(&self.manager); - ScrubWorkerState::Running(iterator) + if let Err(e) = self + .persister + .set_with(|x| x.checkpoint = Some(iterator.clone())) + { + error!("Could not save scrub checkpoint: {}", e); + } + ScrubWorkerState::Running { + iterator, + t_cp: now_msec(), + } } work => { error!("Cannot start scrub worker: already running!"); @@ -292,8 +337,18 @@ impl ScrubWorker { } ScrubWorkerCommand::Pause(dur) => { self.work = match std::mem::take(&mut self.work) { - ScrubWorkerState::Running(it) | ScrubWorkerState::Paused(it, _) => { - ScrubWorkerState::Paused(it, now_msec() + dur.as_millis() as u64) + ScrubWorkerState::Running { iterator, .. } + | ScrubWorkerState::Paused { iterator, .. } => { + if let Err(e) = self + .persister + .set_with(|x| x.checkpoint = Some(iterator.clone())) + { + error!("Could not save scrub checkpoint: {}", e); + } + ScrubWorkerState::Paused { + iterator, + t_resume: now_msec() + dur.as_millis() as u64, + } } work => { error!("Cannot pause scrub worker: not running!"); @@ -303,7 +358,10 @@ impl ScrubWorker { } ScrubWorkerCommand::Resume => { self.work = match std::mem::take(&mut self.work) { - ScrubWorkerState::Paused(it, _) => ScrubWorkerState::Running(it), + ScrubWorkerState::Paused { iterator, .. } => ScrubWorkerState::Running { + iterator, + t_cp: now_msec(), + }, work => { error!("Cannot resume scrub worker: not paused!"); work @@ -312,7 +370,10 @@ impl ScrubWorker { } ScrubWorkerCommand::Cancel => { self.work = match std::mem::take(&mut self.work) { - ScrubWorkerState::Running(_) | ScrubWorkerState::Paused(_, _) => { + ScrubWorkerState::Running { .. } | ScrubWorkerState::Paused { .. } => { + if let Err(e) = self.persister.set_with(|x| x.checkpoint = None) { + error!("Could not save scrub checkpoint: {}", e); + } ScrubWorkerState::Finished } work => { @@ -348,12 +409,15 @@ impl Worker for ScrubWorker { ..Default::default() }; match &self.work { - ScrubWorkerState::Running(bsi) => { - s.progress = Some(format!("{:.2}%", bsi.progress() * 100.)); + ScrubWorkerState::Running { iterator, .. } => { + s.progress = Some(format!("{:.2}%", iterator.progress() * 100.)); } - ScrubWorkerState::Paused(bsi, rt) => { - s.progress = Some(format!("{:.2}%", bsi.progress() * 100.)); - s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))]; + ScrubWorkerState::Paused { iterator, t_resume } => { + s.progress = Some(format!("{:.2}%", iterator.progress() * 100.)); + s.freeform = vec![format!( + "Scrub paused, resumes at {}", + msec_to_rfc3339(*t_resume) + )]; } ScrubWorkerState::Finished => { s.freeform = vec![ @@ -379,9 +443,11 @@ impl Worker for ScrubWorker { }; match &mut self.work { - ScrubWorkerState::Running(bsi) => { + ScrubWorkerState::Running { iterator, t_cp } => { self.tranquilizer.reset(); - if let Some(hash) = bsi.next().await? { + let now = now_msec(); + + if let Some((_path, hash)) = iterator.next().await? { match self.manager.read_block(&hash).await { Err(Error::CorruptData(_)) => { error!("Found corrupt data block during scrub: {:?}", hash); @@ -390,16 +456,32 @@ impl Worker for ScrubWorker { Err(e) => return Err(e), _ => (), }; + + if now - *t_cp > 60 * 1000 { + self.persister + .set_with(|p| p.checkpoint = Some(iterator.clone()))?; + *t_cp = now; + } + Ok(self .tranquilizer .tranquilize_worker(self.persister.get_with(|p| p.tranquility))) } else { + let next_scrub_timestamp = randomize_next_scrub_run_time(now); + self.persister.set_with(|p| { - p.time_last_complete_scrub = now_msec(); - p.time_next_run_scrub = randomize_next_scrub_run_time(now_msec()); + p.time_last_complete_scrub = now; + p.time_next_run_scrub = next_scrub_timestamp; + p.checkpoint = None; })?; self.work = ScrubWorkerState::Finished; self.tranquilizer.clear(); + + info!( + "Datastore scrub completed, next scrub scheduled for {}", + msec_to_rfc3339(next_scrub_timestamp) + ); + Ok(WorkerState::Idle) } } @@ -409,8 +491,8 @@ impl Worker for ScrubWorker { async fn wait_for_work(&mut self) -> WorkerState { let (wait_until, command) = match &self.work { - ScrubWorkerState::Running(_) => return WorkerState::Busy, - ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume), + ScrubWorkerState::Running { .. } => return WorkerState::Busy, + ScrubWorkerState::Paused { t_resume, .. } => (*t_resume, ScrubWorkerCommand::Resume), ScrubWorkerState::Finished => ( self.persister.get_with(|p| p.time_next_run_scrub), ScrubWorkerCommand::Start, @@ -433,110 +515,250 @@ impl Worker for ScrubWorker { } match &self.work { - ScrubWorkerState::Running(_) => WorkerState::Busy, + ScrubWorkerState::Running { .. } => WorkerState::Busy, _ => WorkerState::Idle, } } } +// ---- ---- ---- +// THIRD KIND OF REPAIR: REBALANCING DATA BLOCKS +// between multiple storage locations. +// This is a one-shot repair operation that can be launched, +// checks everything, and then exits. +// ---- ---- ---- + +pub struct RebalanceWorker { + manager: Arc, + block_iter: BlockStoreIterator, + t_started: u64, + t_finished: Option, + moved: usize, + moved_bytes: u64, +} + +impl RebalanceWorker { + pub fn new(manager: Arc) -> Self { + let block_iter = BlockStoreIterator::new(&manager); + Self { + manager, + block_iter, + t_started: now_msec(), + t_finished: None, + moved: 0, + moved_bytes: 0, + } + } +} + +#[async_trait] +impl Worker for RebalanceWorker { + fn name(&self) -> String { + "Block rebalance worker".into() + } + + fn status(&self) -> WorkerStatus { + let t_cur = self.t_finished.unwrap_or_else(|| now_msec()); + let rate = self.moved_bytes / std::cmp::max(1, (t_cur - self.t_started) / 1000); + let mut freeform = vec![ + format!("Blocks moved: {}", self.moved), + format!( + "Bytes moved: {} ({}/s)", + bytesize::ByteSize::b(self.moved_bytes), + bytesize::ByteSize::b(rate) + ), + format!("Started: {}", msec_to_rfc3339(self.t_started)), + ]; + if let Some(t_fin) = self.t_finished { + freeform.push(format!("Finished: {}", msec_to_rfc3339(t_fin))) + } + WorkerStatus { + progress: Some(format!("{:.2}%", self.block_iter.progress() * 100.)), + freeform, + ..Default::default() + } + } + + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + if let Some((path, hash)) = self.block_iter.next().await? { + let prim_loc = self.manager.data_layout.load().primary_block_dir(&hash); + if path.ancestors().all(|x| x != prim_loc) { + let block_path = match path.extension() { + None => DataBlockPath::plain(path.clone()), + Some(x) if x.to_str() == Some("zst") => DataBlockPath::compressed(path.clone()), + _ => { + warn!("not rebalancing file: {}", path.to_string_lossy()); + return Ok(WorkerState::Busy); + } + }; + // block is not in its primary location, + // move it there (reading and re-writing does the trick) + debug!("rebalance: moving block {:?} => {:?}", block_path, prim_loc); + let block_len = self.manager.fix_block_location(&hash, block_path).await?; + self.moved += 1; + self.moved_bytes += block_len as u64; + } + Ok(WorkerState::Busy) + } else { + // all blocks are in their primary location: + // - the ones we moved now are + // - the ones written in the meantime always were, because we only + // write to primary locations + // so we can safely remove all secondary locations from the data layout + let new_layout = self + .manager + .data_layout + .load_full() + .without_secondary_locations(); + self.manager + .data_layout_persister + .save_async(&new_layout) + .await?; + self.manager.data_layout.store(Arc::new(new_layout)); + self.t_finished = Some(now_msec()); + Ok(WorkerState::Done) + } + } + + async fn wait_for_work(&mut self) -> WorkerState { + unreachable!() + } +} + // ---- ---- ---- // UTILITY FOR ENUMERATING THE BLOCK STORE // ---- ---- ---- -struct BlockStoreIterator { - path: Vec, -} - -enum ReadingDir { - Pending(PathBuf), - Read { - subpaths: Vec, - pos: usize, - }, -} +const PROGRESS_FP: u64 = 1_000_000_000; impl BlockStoreIterator { fn new(manager: &BlockManager) -> Self { - let root_dir = manager.data_dir.clone(); - Self { - path: vec![ReadingDir::Pending(root_dir)], + let data_layout = manager.data_layout.load_full(); + + let mut dir_cap = vec![0; data_layout.data_dirs.len()]; + for prim in data_layout.part_prim.iter() { + dir_cap[*prim as usize] += 1; } + for sec_vec in data_layout.part_sec.iter() { + for sec in sec_vec.iter() { + dir_cap[*sec as usize] += 1; + } + } + let sum_cap = dir_cap.iter().sum::() as u64; + + let mut cum_cap = 0; + let mut todo = vec![]; + for (dir, cap) in data_layout.data_dirs.iter().zip(dir_cap.into_iter()) { + let progress_min = (cum_cap * PROGRESS_FP) / sum_cap; + let progress_max = ((cum_cap + cap as u64) * PROGRESS_FP) / sum_cap; + cum_cap += cap as u64; + + todo.push(BsiTodo::Directory { + path: dir.path.clone(), + progress_min, + progress_max, + }); + } + // entries are processed back-to-front (because of .pop()), + // so reverse entries to process them in increasing progress bounds + todo.reverse(); + + let ret = Self { todo }; + debug_assert!(ret.progress_invariant()); + + ret } /// Returns progress done, between 0 and 1 fn progress(&self) -> f32 { - if self.path.is_empty() { - 1.0 - } else { - let mut ret = 0.0; - let mut next_div = 1; - for p in self.path.iter() { - match p { - ReadingDir::Pending(_) => break, - ReadingDir::Read { subpaths, pos } => { - next_div *= subpaths.len(); - ret += ((*pos - 1) as f32) / (next_div as f32); + self.todo + .last() + .map(|x| match x { + BsiTodo::Directory { progress_min, .. } => *progress_min, + BsiTodo::File { progress, .. } => *progress, + }) + .map(|x| x as f32 / PROGRESS_FP as f32) + .unwrap_or(1.0) + } + + async fn next(&mut self) -> Result, Error> { + loop { + match self.todo.pop() { + None => return Ok(None), + Some(BsiTodo::Directory { + path, + progress_min, + progress_max, + }) => { + let istart = self.todo.len(); + + let mut reader = fs::read_dir(&path).await?; + while let Some(ent) = reader.next_entry().await? { + let name = if let Ok(n) = ent.file_name().into_string() { + n + } else { + continue; + }; + let ft = ent.file_type().await?; + if ft.is_dir() && hex::decode(&name).is_ok() { + self.todo.push(BsiTodo::Directory { + path: ent.path(), + progress_min: 0, + progress_max: 0, + }); + } else if ft.is_file() { + let filename = name.split_once('.').map(|(f, _)| f).unwrap_or(&name); + if filename.len() == 64 { + if let Ok(h) = hex::decode(filename) { + let mut hash = [0u8; 32]; + hash.copy_from_slice(&h); + self.todo.push(BsiTodo::File { + path: ent.path(), + hash: hash.into(), + progress: 0, + }); + } + } + } } + + let count = self.todo.len() - istart; + for (i, ent) in self.todo[istart..].iter_mut().enumerate() { + let p1 = progress_min + + ((progress_max - progress_min) * i as u64) / count as u64; + let p2 = progress_min + + ((progress_max - progress_min) * (i + 1) as u64) / count as u64; + match ent { + BsiTodo::Directory { + progress_min, + progress_max, + .. + } => { + *progress_min = p1; + *progress_max = p2; + } + BsiTodo::File { progress, .. } => { + *progress = p1; + } + } + } + self.todo[istart..].reverse(); + debug_assert!(self.progress_invariant()); + } + Some(BsiTodo::File { path, hash, .. }) => { + return Ok(Some((path, hash))); } } - ret } } - async fn next(&mut self) -> Result, Error> { - loop { - let last_path = match self.path.last_mut() { - None => return Ok(None), - Some(lp) => lp, - }; - - if let ReadingDir::Pending(path) = last_path { - let mut reader = fs::read_dir(&path).await?; - let mut subpaths = vec![]; - while let Some(ent) = reader.next_entry().await? { - subpaths.push(ent); - } - *last_path = ReadingDir::Read { subpaths, pos: 0 }; - } - - let (subpaths, pos) = match *last_path { - ReadingDir::Read { - ref subpaths, - ref mut pos, - } => (subpaths, pos), - ReadingDir::Pending(_) => unreachable!(), - }; - - let data_dir_ent = match subpaths.get(*pos) { - None => { - self.path.pop(); - continue; - } - Some(ent) => { - *pos += 1; - ent - } - }; - - let name = data_dir_ent.file_name(); - let name = if let Ok(n) = name.into_string() { - n - } else { - continue; - }; - let ent_type = data_dir_ent.file_type().await?; - - let name = name.strip_suffix(".zst").unwrap_or(&name); - if name.len() == 2 && hex::decode(name).is_ok() && ent_type.is_dir() { - let path = data_dir_ent.path(); - self.path.push(ReadingDir::Pending(path)); - } else if name.len() == 64 { - if let Ok(h) = hex::decode(name) { - let mut hash = [0u8; 32]; - hash.copy_from_slice(&h); - return Ok(Some(hash.into())); - } - } - } + // for debug_assert! + fn progress_invariant(&self) -> bool { + let iter = self.todo.iter().map(|x| match x { + BsiTodo::Directory { progress_min, .. } => progress_min, + BsiTodo::File { progress, .. } => progress, + }); + let iter_1 = iter.clone().skip(1); + iter.zip(iter_1).all(|(prev, next)| prev >= next) } } diff --git a/src/block/resync.rs b/src/block/resync.rs index ea280ad4..7056a828 100644 --- a/src/block/resync.rs +++ b/src/block/resync.rs @@ -15,7 +15,6 @@ use opentelemetry::{ }; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; use garage_util::background::*; use garage_util::data::*; @@ -41,15 +40,15 @@ pub(crate) const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60); pub(crate) const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6; // No more than 4 resync workers can be running in the system -pub(crate) const MAX_RESYNC_WORKERS: usize = 4; +pub(crate) const MAX_RESYNC_WORKERS: usize = 8; // Resync tranquility is initially set to 2, but can be changed in the CLI // and the updated version is persisted over Garage restarts const INITIAL_RESYNC_TRANQUILITY: u32 = 2; pub struct BlockResyncManager { - pub(crate) queue: CountedTree, + pub(crate) queue: db::Tree, pub(crate) notify: Arc, - pub(crate) errors: CountedTree, + pub(crate) errors: db::Tree, busy_set: BusySet, @@ -90,12 +89,10 @@ impl BlockResyncManager { let queue = db .open_tree("block_local_resync_queue") .expect("Unable to open block_local_resync_queue tree"); - let queue = CountedTree::new(queue).expect("Could not count block_local_resync_queue"); let errors = db .open_tree("block_local_resync_errors") .expect("Unable to open block_local_resync_errors tree"); - let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors"); let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg"); @@ -108,18 +105,14 @@ impl BlockResyncManager { } } - /// Get lenght of resync queue - pub fn queue_len(&self) -> Result { - // This currently can't return an error because the CountedTree hack - // doesn't error on .len(), but this will change when we remove the hack - // (hopefully someday!) - Ok(self.queue.len()) + /// Get length of resync queue + pub fn queue_approximate_len(&self) -> Result { + Ok(self.queue.approximate_len()?) } /// Get number of blocks that have an error - pub fn errors_len(&self) -> Result { - // (see queue_len comment) - Ok(self.errors.len()) + pub fn errors_approximate_len(&self) -> Result { + Ok(self.errors.approximate_len()?) } /// Clear the error counter for a block and put it in queue immediately @@ -140,6 +133,14 @@ impl BlockResyncManager { ))) } + /// Clear the entire resync queue and list of errored blocks + /// Corresponds to `garage repair clear-resync-queue` + pub fn clear_resync_queue(&self) -> Result<(), Error> { + self.queue.clear()?; + self.errors.clear()?; + Ok(()) + } + pub fn register_bg_vars(&self, vars: &mut vars::BgVars) { let notify = self.notify.clone(); vars.register_rw( @@ -180,7 +181,7 @@ impl BlockResyncManager { // deleted once the garbage collection delay has passed. // // Here are some explanations on how the resync queue works. - // There are two Sled trees that are used to have information + // There are two db trees that are used to have information // about the status of blocks that need to be resynchronized: // // - resync.queue: a tree that is ordered first by a timestamp @@ -192,10 +193,10 @@ impl BlockResyncManager { // // - resync.errors: a tree that indicates for each block // if the last resync resulted in an error, and if so, - // the following two informations (see the ErrorCounter struct): + // the following two information (see the ErrorCounter struct): // - how many consecutive resync errors for this block? // - when was the last try? - // These two informations are used to implement an + // These two information are used to implement an // exponential backoff retry strategy. // The key in this tree is the 32-byte hash of the block, // and the value is the encoded ErrorCounter value. @@ -359,22 +360,35 @@ impl BlockResyncManager { } async fn resync_block(&self, manager: &BlockManager, hash: &Hash) -> Result<(), Error> { - let BlockStatus { exists, needed } = manager.check_block_status(hash).await?; + let existing_path = manager.find_block(hash).await; + let exists = existing_path.is_some(); + let rc = manager.rc.get_block_rc(hash)?; - if exists != needed.is_needed() || exists != needed.is_nonzero() { + if exists != rc.is_needed() || exists != rc.is_nonzero() { debug!( "Resync block {:?}: exists {}, nonzero rc {}, deletable {}", hash, exists, - needed.is_nonzero(), - needed.is_deletable(), + rc.is_nonzero(), + rc.is_deletable(), ); } - if exists && needed.is_deletable() { - info!("Resync block {:?}: offloading and deleting", hash); + if exists && rc.is_deletable() { + if manager.rc.recalculate_rc(hash)?.0 > 0 { + return Err(Error::Message(format!( + "Refcount for block {:?} was inconsistent, retrying later", + hash + ))); + } - let mut who = manager.replication.write_nodes(hash); + info!("Resync block {:?}: offloading and deleting", hash); + let existing_path = existing_path.unwrap(); + + let mut who = manager + .system + .cluster_layout() + .current_storage_nodes_of(hash); if who.len() < manager.replication.write_quorum() { return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string())); } @@ -382,7 +396,7 @@ impl BlockResyncManager { let who_needs_resps = manager .system - .rpc + .rpc_helper() .call_many( &manager.endpoint, &who, @@ -419,7 +433,7 @@ impl BlockResyncManager { .add(1, &[KeyValue::new("to", format!("{:?}", node))]); } - let block = manager.read_block(hash).await?; + let block = manager.read_block_from(hash, &existing_path).await?; let (header, bytes) = block.into_parts(); let put_block_message = Req::new(BlockRpc::PutBlock { hash: *hash, @@ -428,12 +442,12 @@ impl BlockResyncManager { .with_stream_from_buffer(bytes); manager .system - .rpc + .rpc_helper() .try_call_many( &manager.endpoint, - &need_nodes[..], + &need_nodes, put_block_message, - RequestStrategy::with_priority(PRIO_BACKGROUND) + RequestStrategy::with_priority(PRIO_BACKGROUND | PRIO_SECONDARY) .with_quorum(need_nodes.len()), ) .await @@ -451,13 +465,42 @@ impl BlockResyncManager { manager.rc.clear_deleted_block_rc(hash)?; } - if needed.is_nonzero() && !exists { + if rc.is_nonzero() && !exists { + // The refcount is > 0, and the block is not present locally. + // We might need to fetch it from another node. + + // First, check whether we are still supposed to store that + // block in the latest cluster layout version. + let storage_nodes = manager + .system + .cluster_layout() + .current_storage_nodes_of(&hash); + + if !storage_nodes.contains(&manager.system.id) { + info!( + "Resync block {:?}: block is absent with refcount > 0, but it will drop to zero after all metadata is synced. Not fetching the block.", + hash + ); + return Ok(()); + } + + // We know we need the block. Fetch it. info!( "Resync block {:?}: fetching absent but needed block (refcount > 0)", hash ); - let block_data = manager.rpc_get_raw_block(hash, None).await?; + let block_data = manager + .rpc_get_raw_block(hash, PRIO_BACKGROUND | PRIO_SECONDARY, None) + .await; + if matches!(block_data, Err(Error::MissingBlock(_))) { + warn!( + "Could not fetch needed block {:?}, no node returned valid data. Checking that refcount is correct.", + hash + ); + manager.rc.recalculate_rc(hash)?; + } + let block_data = block_data?; manager.metrics.resync_recv_counter.add(1); @@ -513,9 +556,11 @@ impl Worker for ResyncWorker { } WorkerStatus { - queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64), + queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64), tranquility: Some(tranquility), - persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64), + persistent_errors: Some( + self.manager.resync.errors_approximate_len().unwrap_or(0) as u64 + ), ..Default::default() } } @@ -538,9 +583,9 @@ impl Worker for ResyncWorker { Ok(WorkerState::Idle) } Err(e) => { - // The errors that we have here are only Sled errors + // The errors that we have here are only db errors // We don't really know how to handle them so just ¯\_(ツ)_/¯ - // (there is kind of an assumption that Sled won't error on us, + // (there is kind of an assumption that the db won't error on us, // if it does there is not much we can do -- TODO should we just panic?) // Here we just give the error to the worker manager, // it will print it to the logs and increment a counter diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index e3a65857..9e860e7d 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -11,30 +11,25 @@ readme = "../../README.md" [lib] path = "lib.rs" -[[bin]] -name = "convert" -path = "bin/convert.rs" -required-features = ["cli"] - [dependencies] -err-derive = "0.3" -hexdump = "0.1" -tracing = "0.1" +thiserror.workspace = true +tracing.workspace = true -heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } -rusqlite = { version = "0.28", optional = true } -sled = { version = "0.34", optional = true } +heed = { workspace = true, optional = true } -# cli deps -clap = { version = "4.1", optional = true, features = ["derive", "env"] } -pretty_env_logger = { version = "0.4", optional = true } +rusqlite = { workspace = true, optional = true, features = ["backup"] } +r2d2 = { workspace = true, optional = true } +r2d2_sqlite = { workspace = true, optional = true } + +fjall = { workspace = true, optional = true } +parking_lot = { workspace = true, optional = true } [dev-dependencies] -mktemp = "0.5" +mktemp.workspace = true [features] -default = [ "sled" ] -bundled-libs = [ "rusqlite/bundled" ] -cli = ["clap", "pretty_env_logger"] +default = [ "lmdb", "sqlite" ] +bundled-libs = [ "rusqlite?/bundled" ] lmdb = [ "heed" ] -sqlite = [ "rusqlite" ] +fjall = [ "dep:fjall", "dep:parking_lot" ] +sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ] diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs deleted file mode 100644 index bbde2048..00000000 --- a/src/db/bin/convert.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::path::PathBuf; - -use garage_db::*; - -use clap::Parser; - -/// K2V command line interface -#[derive(Parser, Debug)] -#[clap(author, version, about, long_about = None)] -struct Args { - /// Input DB path - #[clap(short = 'i')] - input_path: PathBuf, - /// Input DB engine - #[clap(short = 'a')] - input_engine: String, - - /// Output DB path - #[clap(short = 'o')] - output_path: PathBuf, - /// Output DB engine - #[clap(short = 'b')] - output_engine: String, -} - -fn main() { - let args = Args::parse(); - pretty_env_logger::init(); - - match do_conversion(args) { - Ok(()) => println!("Success!"), - Err(e) => eprintln!("Error: {}", e), - } -} - -fn do_conversion(args: Args) -> Result<()> { - let input = open_db(args.input_path, args.input_engine)?; - let output = open_db(args.output_path, args.output_engine)?; - output.import(&input)?; - Ok(()) -} - -fn open_db(path: PathBuf, engine: String) -> Result { - match engine.as_str() { - "sled" => { - let db = sled_adapter::sled::Config::default().path(&path).open()?; - Ok(sled_adapter::SledDb::init(db)) - } - "sqlite" | "sqlite3" | "rusqlite" => { - let db = sqlite_adapter::rusqlite::Connection::open(&path)?; - Ok(sqlite_adapter::SqliteDb::init(db)) - } - "lmdb" | "heed" => { - std::fs::create_dir_all(&path).map_err(|e| { - Error(format!("Unable to create LMDB data directory: {}", e).into()) - })?; - - let map_size = lmdb_adapter::recommended_map_size(); - - let db = lmdb_adapter::heed::EnvOpenOptions::new() - .max_dbs(100) - .map_size(map_size) - .open(&path) - .unwrap(); - Ok(lmdb_adapter::LmdbDb::init(db)) - } - e => Err(Error(format!("Invalid DB engine: {}", e).into())), - } -} diff --git a/src/db/counted_tree_hack.rs b/src/db/counted_tree_hack.rs deleted file mode 100644 index bbe943a2..00000000 --- a/src/db/counted_tree_hack.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! This hack allows a db tree to keep in RAM a counter of the number of entries -//! it contains, which is used to call .len() on it. This is usefull only for -//! the sled backend where .len() otherwise would have to traverse the whole -//! tree to count items. For sqlite and lmdb, this is mostly useless (but -//! hopefully not harmfull!). Note that a CountedTree cannot be part of a -//! transaction. - -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; - -use crate::{Result, Tree, TxError, Value, ValueIter}; - -#[derive(Clone)] -pub struct CountedTree(Arc); - -struct CountedTreeInternal { - tree: Tree, - len: AtomicUsize, -} - -impl CountedTree { - pub fn new(tree: Tree) -> Result { - let len = tree.len()?; - Ok(Self(Arc::new(CountedTreeInternal { - tree, - len: AtomicUsize::new(len), - }))) - } - - pub fn len(&self) -> usize { - self.0.len.load(Ordering::SeqCst) - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn get>(&self, key: K) -> Result> { - self.0.tree.get(key) - } - - pub fn first(&self) -> Result> { - self.0.tree.first() - } - - pub fn iter(&self) -> Result> { - self.0.tree.iter() - } - - // ---- writing functions ---- - - pub fn insert(&self, key: K, value: V) -> Result> - where - K: AsRef<[u8]>, - V: AsRef<[u8]>, - { - let old_val = self.0.tree.insert(key, value)?; - if old_val.is_none() { - self.0.len.fetch_add(1, Ordering::SeqCst); - } - Ok(old_val) - } - - pub fn remove>(&self, key: K) -> Result> { - let old_val = self.0.tree.remove(key)?; - if old_val.is_some() { - self.0.len.fetch_sub(1, Ordering::SeqCst); - } - Ok(old_val) - } - - pub fn compare_and_swap( - &self, - key: K, - expected_old: Option, - new: Option, - ) -> Result - where - K: AsRef<[u8]>, - OV: AsRef<[u8]>, - NV: AsRef<[u8]>, - { - let old_some = expected_old.is_some(); - let new_some = new.is_some(); - - let tx_res = self.0.tree.db().transaction(|mut tx| { - let old_val = tx.get(&self.0.tree, &key)?; - let is_same = match (&old_val, &expected_old) { - (None, None) => true, - (Some(x), Some(y)) if x == y.as_ref() => true, - _ => false, - }; - if is_same { - match &new { - Some(v) => { - tx.insert(&self.0.tree, &key, v)?; - } - None => { - tx.remove(&self.0.tree, &key)?; - } - } - tx.commit(()) - } else { - tx.abort(()) - } - }); - - match tx_res { - Ok(()) => { - match (old_some, new_some) { - (false, true) => { - self.0.len.fetch_add(1, Ordering::SeqCst); - } - (true, false) => { - self.0.len.fetch_sub(1, Ordering::SeqCst); - } - _ => (), - } - Ok(true) - } - Err(TxError::Abort(())) => Ok(false), - Err(TxError::Db(e)) => Err(e), - } - } -} diff --git a/src/db/fjall_adapter.rs b/src/db/fjall_adapter.rs new file mode 100644 index 00000000..25913a1f --- /dev/null +++ b/src/db/fjall_adapter.rs @@ -0,0 +1,453 @@ +use core::ops::Bound; + +use std::path::PathBuf; +use std::sync::Arc; + +use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard}; + +use fjall::{ + PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle, + WriteTransaction, +}; + +use crate::{ + open::{Engine, OpenOpt}, + Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, + TxResult, TxValueIter, Value, ValueIter, +}; + +pub use fjall; + +// -- + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening Fjall database at: {}", path.display()); + if opt.fsync { + return Err(Error( + "metadata_fsync is not supported with the Fjall database engine".into(), + )); + } + let mut config = fjall::Config::new(path); + if let Some(block_cache_size) = opt.fjall_block_cache_size { + config = config.cache_size(block_cache_size as u64); + } + let keyspace = config.open_transactional()?; + Ok(FjallDb::init(keyspace)) +} + +// -- err + +impl From for Error { + fn from(e: fjall::Error) -> Error { + Error(format!("fjall: {}", e).into()) + } +} + +impl From for Error { + fn from(e: fjall::LsmError) -> Error { + Error(format!("fjall lsm_tree: {}", e).into()) + } +} + +impl From for TxOpError { + fn from(e: fjall::Error) -> TxOpError { + TxOpError(e.into()) + } +} + +// -- db + +pub struct FjallDb { + keyspace: TransactionalKeyspace, + trees: RwLock>, +} + +type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>); + +impl FjallDb { + pub fn init(keyspace: TransactionalKeyspace) -> Db { + let s = Self { + keyspace, + trees: RwLock::new(Vec::new()), + }; + Db(Arc::new(s)) + } + + fn get_tree( + &self, + i: usize, + ) -> Result> { + RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| { + trees.get(i).map(|tup| &tup.1) + }) + .map_err(|_| Error("invalid tree id".into())) + } +} + +impl IDb for FjallDb { + fn engine(&self) -> String { + "Fjall (EXPERIMENTAL!)".into() + } + + fn open_tree(&self, name: &str) -> Result { + let mut trees = self.trees.write(); + let safe_name = encode_name(name)?; + if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) { + Ok(i) + } else { + let tree = self + .keyspace + .open_partition(&safe_name, PartitionCreateOptions::default())?; + let i = trees.len(); + trees.push((safe_name, tree)); + Ok(i) + } + } + + fn list_trees(&self) -> Result> { + Ok(self + .keyspace + .list_partitions() + .iter() + .map(|n| decode_name(&n)) + .collect::>>()?) + } + + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Fjall.db_path(base_path); + + let source_state = self.keyspace.read_tx(); + let copy_keyspace = fjall::Config::new(path).open()?; + + for partition_name in self.keyspace.list_partitions() { + let source_partition = self + .keyspace + .open_partition(&partition_name, PartitionCreateOptions::default())?; + let copy_partition = + copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?; + + for entry in source_state.iter(&source_partition) { + let (key, value) = entry?; + copy_partition.insert(key, value)?; + } + } + + copy_keyspace.persist(PersistMode::SyncAll)?; + Ok(()) + } + + // ---- + + fn get(&self, tree_idx: usize, key: &[u8]) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + let val = tx.get(&tree, key)?; + match val { + None => Ok(None), + Some(v) => Ok(Some(v.to_vec())), + } + } + + fn approximate_len(&self, tree_idx: usize) -> Result { + let tree = self.get_tree(tree_idx)?; + Ok(tree.approximate_len()) + } + fn is_empty(&self, tree_idx: usize) -> Result { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(tx.is_empty(&tree)?) + } + + fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree_idx)?; + let mut tx = self.keyspace.write_tx(); + tx.insert(&tree, key, value); + tx.commit()?; + Ok(()) + } + + fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> { + let tree = self.get_tree(tree_idx)?; + let mut tx = self.keyspace.write_tx(); + tx.remove(&tree, key); + tx.commit()?; + Ok(()) + } + + fn clear(&self, tree_idx: usize) -> Result<()> { + let mut trees = self.trees.write(); + + if tree_idx >= trees.len() { + return Err(Error("invalid tree id".into())); + } + let (name, tree) = trees.remove(tree_idx); + + self.keyspace.delete_partition(tree)?; + let tree = self + .keyspace + .open_partition(&name, PartitionCreateOptions::default())?; + trees.insert(tree_idx, (name, tree)); + + Ok(()) + } + + fn iter(&self, tree_idx: usize) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new(tx.iter(&tree).map(iterator_remap))) + } + + fn iter_rev(&self, tree_idx: usize) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap))) + } + + fn range<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new( + tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high)) + .map(iterator_remap), + )) + } + fn range_rev<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree_idx)?; + let tx = self.keyspace.read_tx(); + Ok(Box::new( + tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high)) + .rev() + .map(iterator_remap), + )) + } + + // ---- + + fn transaction(&self, f: &dyn ITxFn) -> TxResult { + let trees = self.trees.read(); + let mut tx = FjallTx { + trees: &trees[..], + tx: self.keyspace.write_tx(), + }; + + let res = f.try_on(&mut tx); + match res { + TxFnResult::Ok(on_commit) => { + tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; + Ok(on_commit) + } + TxFnResult::Abort => { + tx.tx.rollback(); + Err(TxError::Abort(())) + } + TxFnResult::DbErr => { + tx.tx.rollback(); + Err(TxError::Db(Error( + "(this message will be discarded)".into(), + ))) + } + } + } +} + +// ---- + +struct FjallTx<'a> { + trees: &'a [(String, TransactionalPartitionHandle)], + tx: WriteTransaction<'a>, +} + +impl<'a> FjallTx<'a> { + fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> { + self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| { + TxOpError(Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + )) + }) + } +} + +impl<'a> ITx for FjallTx<'a> { + fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + match self.tx.get(tree, key)? { + Some(v) => Ok(Some(v.to_vec())), + None => Ok(None), + } + } + fn len(&self, tree_idx: usize) -> TxOpResult { + let tree = self.get_tree(tree_idx)?; + Ok(self.tx.len(tree)? as usize) + } + + fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { + let tree = self.get_tree(tree_idx)?.clone(); + self.tx.insert(&tree, key, value); + Ok(()) + } + fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> { + let tree = self.get_tree(tree_idx)?.clone(); + self.tx.remove(&tree, key); + Ok(()) + } + fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> { + unimplemented!("LSM tree clearing in cross-partition transaction is not supported") + } + + fn iter(&self, tree_idx: usize) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?.clone(); + Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx))) + } + fn iter_rev(&self, tree_idx: usize) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?.clone(); + Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx))) + } + + fn range<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + let low = clone_bound(low); + let high = clone_bound(high); + Ok(Box::new( + self.tx + .range::, ByteVecRangeBounds>(&tree, (low, high)) + .map(iterator_remap_tx), + )) + } + fn range_rev<'r>( + &self, + tree_idx: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> TxOpResult> { + let tree = self.get_tree(tree_idx)?; + let low = clone_bound(low); + let high = clone_bound(high); + Ok(Box::new( + self.tx + .range::, ByteVecRangeBounds>(&tree, (low, high)) + .rev() + .map(iterator_remap_tx), + )) + } +} + +// -- maps fjall's (k, v) to ours + +fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> { + r.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| e.into()) +} + +fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> { + r.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| e.into()) +} + +// -- utils to deal with Garage's tightness on Bound lifetimes + +type ByteVecBound = Bound>; +type ByteVecRangeBounds = (ByteVecBound, ByteVecBound); + +fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound { + let value = match bound { + Bound::Excluded(v) | Bound::Included(v) => v.to_vec(), + Bound::Unbounded => vec![], + }; + + match bound { + Bound::Included(_) => Bound::Included(value), + Bound::Excluded(_) => Bound::Excluded(value), + Bound::Unbounded => Bound::Unbounded, + } +} + +// -- utils to encode table names -- + +fn encode_name(s: &str) -> Result { + let base = 'A' as u32; + + let mut ret = String::with_capacity(s.len() + 10); + for c in s.chars() { + if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' { + ret.push(c); + } else if c <= u8::MAX as char { + ret.push('$'); + let c_hi = c as u32 / 16; + let c_lo = c as u32 % 16; + ret.push(char::from_u32(base + c_hi).unwrap()); + ret.push(char::from_u32(base + c_lo).unwrap()); + } else { + return Err(Error( + format!("table name {} could not be safely encoded", s).into(), + )); + } + } + Ok(ret) +} + +fn decode_name(s: &str) -> Result { + use std::convert::TryFrom; + + let errfn = || Error(format!("encoded table name {} is invalid", s).into()); + let c_map = |c: char| { + let c = c as u32; + let base = 'A' as u32; + if (base..base + 16).contains(&c) { + Some(c - base) + } else { + None + } + }; + + let mut ret = String::with_capacity(s.len()); + let mut it = s.chars(); + while let Some(c) = it.next() { + if c == '$' { + let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?; + let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?; + let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?; + ret.push(c_dec); + } else { + ret.push(c); + } + } + Ok(ret) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encdec_name() { + for name in [ + "testname", + "test_name", + "test name", + "test$name", + "test:name@help.me$get/this**right", + ] { + let encname = encode_name(name).unwrap(); + assert!(!encname.contains(' ')); + assert!(!encname.contains('.')); + assert!(!encname.contains('*')); + assert_eq!(*name, decode_name(&encname).unwrap()); + } + } +} diff --git a/src/db/lib.rs b/src/db/lib.rs index 11cae4e3..2a467c7c 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -1,18 +1,14 @@ #[macro_use] -#[cfg(feature = "sqlite")] extern crate tracing; -#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))] -compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite."); - +#[cfg(feature = "fjall")] +pub mod fjall_adapter; #[cfg(feature = "lmdb")] pub mod lmdb_adapter; -#[cfg(feature = "sled")] -pub mod sled_adapter; #[cfg(feature = "sqlite")] pub mod sqlite_adapter; -pub mod counted_tree_hack; +pub mod open; #[cfg(test)] pub mod test; @@ -21,14 +17,22 @@ use core::ops::{Bound, RangeBounds}; use std::borrow::Cow; use std::cell::Cell; +use std::path::PathBuf; use std::sync::Arc; -use err_derive::Error; +use thiserror::Error; + +pub use open::*; + +pub(crate) type OnCommit = Vec>; #[derive(Clone)] pub struct Db(pub(crate) Arc); -pub struct Transaction<'a>(&'a mut dyn ITx); +pub struct Transaction<'a> { + tx: &'a mut dyn ITx, + on_commit: OnCommit, +} #[derive(Clone)] pub struct Tree(Arc, usize); @@ -40,16 +44,23 @@ pub type TxValueIter<'a> = Box); +impl From for Error { + fn from(e: std::io::Error) -> Error { + Error(format!("IO: {}", e).into()) + } +} + pub type Result = std::result::Result; #[derive(Debug, Error)] -#[error(display = "{}", _0)] +#[error("{0}")] pub struct TxOpError(pub(crate) Error); pub type TxOpResult = std::result::Result; +#[derive(Debug)] pub enum TxError { Abort(E), Db(Error), @@ -88,39 +99,58 @@ impl Db { pub fn transaction(&self, fun: F) -> TxResult where - F: Fn(Transaction<'_>) -> TxResult, + F: Fn(&mut Transaction<'_>) -> TxResult, { let f = TxFn { function: fun, result: Cell::new(None), }; let tx_res = self.0.transaction(&f); - let ret = f - .result - .into_inner() - .expect("Transaction did not store result"); + let fn_res = f.result.into_inner(); - match tx_res { - Ok(()) => { - assert!(matches!(ret, Ok(_))); - ret + match (tx_res, fn_res) { + (Ok(on_commit), Some(Ok(value))) => { + // Transaction succeeded + // TxFn stored the value to return to the user in fn_res + // tx_res contains the on_commit list of callbacks, run them now + on_commit.into_iter().for_each(|f| f()); + Ok(value) } - Err(TxError::Abort(())) => { - assert!(matches!(ret, Err(TxError::Abort(_)))); - ret + (Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => { + // Transaction was aborted by user code + // The abort error value is stored in fn_res + Err(TxError::Abort(e)) + } + (Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => { + // Transaction encountered a DB error in user code + // The error value encountered is the one in fn_res, + // tx_res contains only a dummy error message + Err(TxError::Db(fn_e)) + } + (Err(TxError::Db(tx_e)), None) => { + // Transaction encounterred a DB error when initializing the transaction, + // before user code was called + Err(TxError::Db(tx_e)) + } + (Err(TxError::Db(tx_e)), Some(Ok(_))) => { + // Transaction encounterred a DB error when commiting the transaction, + // after user code was called + Err(TxError::Db(tx_e)) + } + (tx_res, fn_res) => { + panic!( + "unexpected error case: tx_res={:?}, fn_res={:?}", + tx_res.map(|_| "..."), + fn_res.map(|x| x.map(|_| "...").map_err(|_| "...")) + ); } - Err(TxError::Db(e2)) => match ret { - // Ok was stored -> the error occured when finalizing - // transaction - Ok(_) => Err(TxError::Db(e2)), - // An error was already stored: that's the one we want to - // return - Err(TxError::Db(e)) => Err(TxError::Db(e)), - _ => unreachable!(), - }, } } + pub fn snapshot(&self, path: &PathBuf) -> Result<()> { + self.0.snapshot(path) + } + pub fn import(&self, other: &Db) -> Result<()> { let existing_trees = self.list_trees()?; if !existing_trees.is_empty() { @@ -136,13 +166,13 @@ impl Db { let tree_names = other.list_trees()?; for name in tree_names { let tree = self.open_tree(&name)?; - if tree.len()? > 0 { + if !tree.is_empty()? { return Err(Error(format!("tree {} already contains data", name).into())); } let ex_tree = other.open_tree(&name)?; - let tx_res = self.transaction(|mut tx| { + let tx_res = self.transaction(|tx| { let mut i = 0; for item in ex_tree.iter().map_err(TxError::Abort)? { let (k, v) = item.map_err(TxError::Abort)?; @@ -152,7 +182,7 @@ impl Db { println!("{}: imported {}", name, i); } } - tx.commit(i) + Ok(i) }); let total = match tx_res { Err(TxError::Db(e)) => return Err(e), @@ -178,12 +208,12 @@ impl Tree { self.0.get(self.1, key.as_ref()) } #[inline] - pub fn len(&self) -> Result { - self.0.len(self.1) + pub fn approximate_len(&self) -> Result { + self.0.approximate_len(self.1) } #[inline] - pub fn fast_len(&self) -> Result> { - self.0.fast_len(self.1) + pub fn is_empty(&self) -> Result { + self.0.is_empty(self.1) } #[inline] @@ -199,16 +229,12 @@ impl Tree { /// Returns the old value if there was one #[inline] - pub fn insert, U: AsRef<[u8]>>( - &self, - key: T, - value: U, - ) -> Result> { + pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } /// Returns the old value if there was one #[inline] - pub fn remove>(&self, key: T) -> Result> { + pub fn remove>(&self, key: T) -> Result<()> { self.0.remove(self.1, key.as_ref()) } /// Clears all values from the tree @@ -252,11 +278,11 @@ impl Tree { impl<'a> Transaction<'a> { #[inline] pub fn get>(&self, tree: &Tree, key: T) -> TxOpResult> { - self.0.get(tree.1, key.as_ref()) + self.tx.get(tree.1, key.as_ref()) } #[inline] pub fn len(&self, tree: &Tree) -> TxOpResult { - self.0.len(tree.1) + self.tx.len(tree.1) } /// Returns the old value if there was one @@ -266,22 +292,27 @@ impl<'a> Transaction<'a> { tree: &Tree, key: T, value: U, - ) -> TxOpResult> { - self.0.insert(tree.1, key.as_ref(), value.as_ref()) + ) -> TxOpResult<()> { + self.tx.insert(tree.1, key.as_ref(), value.as_ref()) } /// Returns the old value if there was one #[inline] - pub fn remove>(&mut self, tree: &Tree, key: T) -> TxOpResult> { - self.0.remove(tree.1, key.as_ref()) + pub fn remove>(&mut self, tree: &Tree, key: T) -> TxOpResult<()> { + self.tx.remove(tree.1, key.as_ref()) + } + /// Clears all values in a tree + #[inline] + pub fn clear(&mut self, tree: &Tree) -> TxOpResult<()> { + self.tx.clear(tree.1) } #[inline] pub fn iter(&self, tree: &Tree) -> TxOpResult> { - self.0.iter(tree.1) + self.tx.iter(tree.1) } #[inline] pub fn iter_rev(&self, tree: &Tree) -> TxOpResult> { - self.0.iter_rev(tree.1) + self.tx.iter_rev(tree.1) } #[inline] @@ -292,7 +323,7 @@ impl<'a> Transaction<'a> { { let sb = range.start_bound(); let eb = range.end_bound(); - self.0.range(tree.1, get_bound(sb), get_bound(eb)) + self.tx.range(tree.1, get_bound(sb), get_bound(eb)) } #[inline] pub fn range_rev(&self, tree: &Tree, range: R) -> TxOpResult> @@ -302,19 +333,12 @@ impl<'a> Transaction<'a> { { let sb = range.start_bound(); let eb = range.end_bound(); - self.0.range_rev(tree.1, get_bound(sb), get_bound(eb)) - } - - // ---- - - #[inline] - pub fn abort(self, e: E) -> TxResult { - Err(TxError::Abort(e)) + self.tx.range_rev(tree.1, get_bound(sb), get_bound(eb)) } #[inline] - pub fn commit(self, r: R) -> TxResult { - Ok(r) + pub fn on_commit(&mut self, f: F) { + self.on_commit.push(Box::new(f)); } } @@ -324,15 +348,14 @@ pub(crate) trait IDb: Send + Sync { fn engine(&self) -> String; fn open_tree(&self, name: &str) -> Result; fn list_trees(&self) -> Result>; + fn snapshot(&self, path: &PathBuf) -> Result<()>; fn get(&self, tree: usize, key: &[u8]) -> Result>; - fn len(&self, tree: usize) -> Result; - fn fast_len(&self, _tree: usize) -> Result> { - Ok(None) - } + fn approximate_len(&self, tree: usize) -> Result; + fn is_empty(&self, tree: usize) -> Result; - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result>; - fn remove(&self, tree: usize, key: &[u8]) -> Result>; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn remove(&self, tree: usize, key: &[u8]) -> Result<()>; fn clear(&self, tree: usize) -> Result<()>; fn iter(&self, tree: usize) -> Result>; @@ -351,15 +374,16 @@ pub(crate) trait IDb: Send + Sync { high: Bound<&'r [u8]>, ) -> Result>; - fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; + fn transaction(&self, f: &dyn ITxFn) -> TxResult; } pub(crate) trait ITx { fn get(&self, tree: usize, key: &[u8]) -> TxOpResult>; fn len(&self, tree: usize) -> TxOpResult; - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult>; - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult>; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()>; + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()>; + fn clear(&mut self, tree: usize) -> TxOpResult<()>; fn iter(&self, tree: usize) -> TxOpResult>; fn iter_rev(&self, tree: usize) -> TxOpResult>; @@ -383,14 +407,14 @@ pub(crate) trait ITxFn { } pub(crate) enum TxFnResult { - Ok, + Ok(OnCommit), Abort, DbErr, } struct TxFn where - F: Fn(Transaction<'_>) -> TxResult, + F: Fn(&mut Transaction<'_>) -> TxResult, { function: F, result: Cell>>, @@ -398,12 +422,16 @@ where impl ITxFn for TxFn where - F: Fn(Transaction<'_>) -> TxResult, + F: Fn(&mut Transaction<'_>) -> TxResult, { fn try_on(&self, tx: &mut dyn ITx) -> TxFnResult { - let res = (self.function)(Transaction(tx)); + let mut tx = Transaction { + tx, + on_commit: vec![], + }; + let res = (self.function)(&mut tx); let res2 = match &res { - Ok(_) => TxFnResult::Ok, + Ok(_) => TxFnResult::Ok(tx.on_commit), Err(TxError::Abort(_)) => TxFnResult::Abort, Err(TxError::Db(_)) => TxFnResult::DbErr, }; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 31956612..ac185ae9 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -1,20 +1,65 @@ use core::ops::Bound; -use core::ptr::NonNull; use std::collections::HashMap; use std::convert::TryInto; +use std::marker::PhantomPinned; +use std::path::PathBuf; +use std::pin::Pin; use std::sync::{Arc, RwLock}; use heed::types::ByteSlice; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ - Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - TxValueIter, Value, ValueIter, + open::{Engine, OpenOpt}, + Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, + TxResult, TxValueIter, Value, ValueIter, }; pub use heed; +// ---- top-level open function + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening LMDB database at: {}", path.display()); + if let Err(e) = std::fs::create_dir_all(&path) { + return Err(Error( + format!("Unable to create LMDB data directory: {}", e).into(), + )); + } + + let map_size = match opt.lmdb_map_size { + None => recommended_map_size(), + Some(v) => v - (v % 4096), + }; + + let mut env_builder = heed::EnvOpenOptions::new(); + env_builder.max_dbs(100); + env_builder.map_size(map_size); + env_builder.max_readers(2048); + unsafe { + env_builder.flag(heed::flags::Flags::MdbNoRdAhead); + env_builder.flag(heed::flags::Flags::MdbNoMetaSync); + if !opt.fsync { + env_builder.flag(heed::flags::Flags::MdbNoSync); + } + } + match env_builder.open(&path) { + Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => { + return Err(Error( + "OutOfMemory error while trying to open LMDB database. This can happen \ + if your operating system is not allowing you to use sufficient virtual \ + memory address space. Please check that no limit is set (ulimit -v). \ + You may also try to set a smaller `lmdb_map_size` configuration parameter. \ + On 32-bit machines, you should probably switch to another database engine." + .into(), + )) + } + Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())), + Ok(db) => Ok(LmdbDb::init(db)), + } +} + // -- err impl From for Error { @@ -102,6 +147,14 @@ impl IDb for LmdbDb { Ok(ret2) } + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Lmdb.db_path(base_path); + self.db + .copy_to_path(path, heed::CompactionOption::Enabled)?; + Ok(()) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result> { @@ -115,32 +168,31 @@ impl IDb for LmdbDb { } } - fn len(&self, tree: usize) -> Result { + fn approximate_len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; Ok(tree.len(&tx)?.try_into().unwrap()) } - - fn fast_len(&self, tree: usize) -> Result> { - Ok(Some(self.len(tree)?)) + fn is_empty(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + Ok(tree.is_empty(&tx)?) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - let old_val = tree.get(&tx, key)?.map(Vec::from); tree.put(&mut tx, key, value)?; tx.commit()?; - Ok(old_val) + Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result> { + fn remove(&self, tree: usize, key: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - let old_val = tree.get(&tx, key)?.map(Vec::from); tree.delete(&mut tx, key)?; tx.commit()?; - Ok(old_val) + Ok(()) } fn clear(&self, tree: usize) -> Result<()> { @@ -154,13 +206,15 @@ impl IDb for LmdbDb { fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) } } fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) } } fn range<'r>( @@ -171,7 +225,8 @@ impl IDb for LmdbDb { ) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) } } fn range_rev<'r>( &self, @@ -181,12 +236,13 @@ impl IDb for LmdbDb { ) -> Result> { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) + // Safety: the cloture does not store its argument anywhere, + unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) } } // ---- - fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { + fn transaction(&self, f: &dyn ITxFn) -> TxResult { let trees = self.trees.read().unwrap(); let mut tx = LmdbTx { trees: &trees.0[..], @@ -199,9 +255,9 @@ impl IDb for LmdbDb { let res = f.try_on(&mut tx); match res { - TxFnResult::Ok => { + TxFnResult::Ok(on_commit) => { tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; - Ok(()) + Ok(on_commit) } TxFnResult::Abort => { tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?; @@ -228,7 +284,7 @@ impl<'a> LmdbTx<'a> { fn get_tree(&self, i: usize) -> TxOpResult<&Database> { self.trees.get(i).ok_or_else(|| { TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), + "invalid tree id (it might have been opened after the transaction started)".into(), )) }) } @@ -242,49 +298,63 @@ impl<'a> ITx for LmdbTx<'a> { None => Ok(None), } } - fn len(&self, _tree: usize) -> TxOpResult { - unimplemented!(".len() in transaction not supported with LMDB backend") + fn len(&self, tree: usize) -> TxOpResult { + let tree = self.get_tree(tree)?; + Ok(tree.len(&self.tx)? as usize) } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { let tree = *self.get_tree(tree)?; - let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.put(&mut self.tx, key, value)?; - Ok(old_val) + Ok(()) } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> { let tree = *self.get_tree(tree)?; - let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.delete(&mut self.tx, key)?; - Ok(old_val) + Ok(()) + } + fn clear(&mut self, tree: usize) -> TxOpResult<()> { + let tree = *self.get_tree(tree)?; + tree.clear(&mut self.tx)?; + Ok(()) } - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + fn iter(&self, tree: usize) -> TxOpResult> { + let tree = *self.get_tree(tree)?; + Ok(Box::new(tree.iter(&self.tx)?.map(tx_iter_item))) } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + fn iter_rev(&self, tree: usize) -> TxOpResult> { + let tree = *self.get_tree(tree)?; + Ok(Box::new(tree.rev_iter(&self.tx)?.map(tx_iter_item))) } fn range<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + let tree = *self.get_tree(tree)?; + Ok(Box::new( + tree.range(&self.tx, &(low, high))?.map(tx_iter_item), + )) } fn range_rev<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with LMDB backend"); + let tree = *self.get_tree(tree)?; + Ok(Box::new( + tree.rev_range(&self.tx, &(low, high))?.map(tx_iter_item), + )) } } -// ---- +// ---- iterators outside transactions ---- +// complicated, they must hold the transaction object +// therefore a bit of unsafe code (it is a self-referential struct) type IteratorItem<'a> = heed::Result<( >::DItem, @@ -297,22 +367,43 @@ where { tx: RoTxn<'a>, iter: Option, + _pin: PhantomPinned, } impl<'a, I> TxAndIterator<'a, I> where I: Iterator> + 'a, { - fn make(tx: RoTxn<'a>, iterfun: F) -> Result> + fn iter(self: Pin<&mut Self>) -> &mut Option { + // Safety: iter is not structural + unsafe { &mut self.get_unchecked_mut().iter } + } + + /// Safety: iterfun must not store its argument anywhere but in its result. + unsafe fn make(tx: RoTxn<'a>, iterfun: F) -> Result> where F: FnOnce(&'a RoTxn<'a>) -> Result, { - let mut res = TxAndIterator { tx, iter: None }; + let res = TxAndIterator { + tx, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); - let tx = unsafe { NonNull::from(&res.tx).as_ref() }; - res.iter = Some(iterfun(tx)?); + let tx_lifetime_overextended: &'a RoTxn<'a> = { + let tx = &boxed.tx; + // Safety: Artificially extending the lifetime because + // this reference will only be stored and accessed from the + // returned ValueIter which guarantees that it is destroyed + // before the tx it is pointing to. + unsafe { &*&raw const *tx } + }; + let iter = iterfun(&tx_lifetime_overextended)?; - Ok(Box::new(res)) + *boxed.as_mut().iter() = Some(iter); + + Ok(Box::new(TxAndIteratorPin(boxed))) } } @@ -321,26 +412,44 @@ where I: Iterator> + 'a, { fn drop(&mut self) { - drop(self.iter.take()); + // Safety: `new_unchecked` is okay because we know this value is never + // used again after being dropped. + let this = unsafe { Pin::new_unchecked(self) }; + drop(this.iter().take()); } } -impl<'a, I> Iterator for TxAndIterator<'a, I> +struct TxAndIteratorPin<'a, I>(Pin>>) +where + I: Iterator> + 'a; + +impl<'a, I> Iterator for TxAndIteratorPin<'a, I> where I: Iterator> + 'a, { type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { - match self.iter.as_mut().unwrap().next() { - None => None, - Some(Err(e)) => Some(Err(e.into())), - Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), - } + let mut_ref = Pin::as_mut(&mut self.0); + let next = mut_ref.iter().as_mut()?.next()?; + let res = match next { + Err(e) => Err(e.into()), + Ok((k, v)) => Ok((k.to_vec(), v.to_vec())), + }; + Some(res) } } -// ---- +// ---- iterators within transactions ---- + +fn tx_iter_item<'a>( + item: std::result::Result<(&'a [u8], &'a [u8]), heed::Error>, +) -> TxOpResult<(Vec, Vec)> { + item.map(|(k, v)| (k.to_vec(), v.to_vec())) + .map_err(|e| TxOpError(Error::from(e))) +} + +// ---- utility ---- #[cfg(target_pointer_width = "64")] pub fn recommended_map_size() -> usize { @@ -349,6 +458,6 @@ pub fn recommended_map_size() -> usize { #[cfg(target_pointer_width = "32")] pub fn recommended_map_size() -> usize { - warn!("LMDB is not recommended on 32-bit systems, database size will be limited"); + tracing::warn!("LMDB is not recommended on 32-bit systems, database size will be limited"); 1usize << 30 } diff --git a/src/db/open.rs b/src/db/open.rs new file mode 100644 index 00000000..23391c61 --- /dev/null +++ b/src/db/open.rs @@ -0,0 +1,109 @@ +use std::path::PathBuf; + +use crate::{Db, Error, Result}; + +/// List of supported database engine types +/// +/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter +/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine +/// and valid engine, whose support is not enabled via feature flag. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Engine { + Lmdb, + Sqlite, + Fjall, +} + +impl Engine { + /// Return variant name as static `&str` + pub fn as_str(&self) -> &'static str { + match self { + Self::Lmdb => "lmdb", + Self::Sqlite => "sqlite", + Self::Fjall => "fjall", + } + } + + /// Return engine-specific DB path from base path + pub fn db_path(&self, base_path: &PathBuf) -> PathBuf { + let mut ret = base_path.clone(); + match self { + Self::Lmdb => { + ret.push("db.lmdb"); + } + Self::Sqlite => { + ret.push("db.sqlite"); + } + Self::Fjall => { + ret.push("db.fjall"); + } + } + ret + } +} + +impl std::fmt::Display for Engine { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.as_str().fmt(fmt) + } +} + +impl std::str::FromStr for Engine { + type Err = Error; + + fn from_str(text: &str) -> Result { + match text { + "lmdb" | "heed" => Ok(Self::Lmdb), + "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite), + "fjall" => Ok(Self::Fjall), + "sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())), + kind => Err(Error( + format!( + "Invalid DB engine: {} (options are: lmdb, sqlite, fjall)", + kind + ) + .into(), + )), + } + } +} + +pub struct OpenOpt { + pub fsync: bool, + pub lmdb_map_size: Option, + pub fjall_block_cache_size: Option, +} + +impl Default for OpenOpt { + fn default() -> Self { + Self { + fsync: false, + lmdb_map_size: None, + fjall_block_cache_size: None, + } + } +} + +pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result { + match engine { + // ---- Sqlite DB ---- + #[cfg(feature = "sqlite")] + Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt), + + // ---- LMDB DB ---- + #[cfg(feature = "lmdb")] + Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt), + + // ---- Fjall DB ---- + #[cfg(feature = "fjall")] + Engine::Fjall => crate::fjall_adapter::open_db(path, opt), + + // Pattern is unreachable when all supported DB engines are compiled into binary. The allow + // attribute is added so that we won't have to change this match in case stop building + // support for one or more engines by default. + #[allow(unreachable_patterns)] + engine => Err(Error( + format!("DB engine support not available in this build: {}", engine).into(), + )), + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs deleted file mode 100644 index cf61867d..00000000 --- a/src/db/sled_adapter.rs +++ /dev/null @@ -1,266 +0,0 @@ -use core::ops::Bound; - -use std::cell::Cell; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use sled::transaction::{ - ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, - UnabortableTransactionError, -}; - -use crate::{ - Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - TxValueIter, Value, ValueIter, -}; - -pub use sled; - -// -- err - -impl From for Error { - fn from(e: sled::Error) -> Error { - Error(format!("Sled: {}", e).into()) - } -} - -impl From for TxOpError { - fn from(e: sled::Error) -> TxOpError { - TxOpError(e.into()) - } -} - -// -- db - -pub struct SledDb { - db: sled::Db, - trees: RwLock<(Vec, HashMap)>, -} - -impl SledDb { - pub fn init(db: sled::Db) -> Db { - let s = Self { - db, - trees: RwLock::new((Vec::new(), HashMap::new())), - }; - Db(Arc::new(s)) - } - - fn get_tree(&self, i: usize) -> Result { - self.trees - .read() - .unwrap() - .0 - .get(i) - .cloned() - .ok_or_else(|| Error("invalid tree id".into())) - } -} - -impl IDb for SledDb { - fn engine(&self) -> String { - "Sled".into() - } - - fn open_tree(&self, name: &str) -> Result { - let mut trees = self.trees.write().unwrap(); - if let Some(i) = trees.1.get(name) { - Ok(*i) - } else { - let tree = self.db.open_tree(name)?; - let i = trees.0.len(); - trees.0.push(tree); - trees.1.insert(name.to_string(), i); - Ok(i) - } - } - - fn list_trees(&self) -> Result> { - let mut trees = vec![]; - for name in self.db.tree_names() { - let name = std::str::from_utf8(&name) - .map_err(|e| Error(format!("{}", e).into()))? - .to_string(); - if name != "__sled__default" { - trees.push(name); - } - } - Ok(trees) - } - - // ---- - - fn get(&self, tree: usize, key: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let val = tree.get(key)?; - Ok(val.map(|x| x.to_vec())) - } - - fn len(&self, tree: usize) -> Result { - let tree = self.get_tree(tree)?; - Ok(tree.len()) - } - - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let old_val = tree.insert(key, value)?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn remove(&self, tree: usize, key: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; - let old_val = tree.remove(key)?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn clear(&self, tree: usize) -> Result<()> { - let tree = self.get_tree(tree)?; - tree.clear()?; - Ok(()) - } - - fn iter(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - - fn iter_rev(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - - fn range<'r>( - &self, - tree: usize, - low: Bound<&'r [u8]>, - high: Bound<&'r [u8]>, - ) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { - v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) - }))) - } - fn range_rev<'r>( - &self, - tree: usize, - low: Bound<&'r [u8]>, - high: Bound<&'r [u8]>, - ) -> Result> { - let tree = self.get_tree(tree)?; - Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( - |v| v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into), - ))) - } - - // ---- - - fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { - let trees = self.trees.read().unwrap(); - let res = trees.0.transaction(|txtrees| { - let mut tx = SledTx { - trees: txtrees, - err: Cell::new(None), - }; - match f.try_on(&mut tx) { - TxFnResult::Ok => { - assert!(tx.err.into_inner().is_none()); - Ok(()) - } - TxFnResult::Abort => { - assert!(tx.err.into_inner().is_none()); - Err(ConflictableTransactionError::Abort(())) - } - TxFnResult::DbErr => { - let e = tx.err.into_inner().expect("No DB error"); - Err(e.into()) - } - } - }); - match res { - Ok(()) => Ok(()), - Err(TransactionError::Abort(())) => Err(TxError::Abort(())), - Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), - } - } -} - -// ---- - -struct SledTx<'a> { - trees: &'a [TransactionalTree], - err: Cell>, -} - -impl<'a> SledTx<'a> { - fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalTree> { - self.trees.get(i).ok_or_else(|| { - TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), - )) - }) - } - - fn save_error( - &self, - v: std::result::Result, - ) -> TxOpResult { - match v { - Ok(x) => Ok(x), - Err(e) => { - let txt = format!("{}", e); - self.err.set(Some(e)); - Err(TxOpError(Error(txt.into()))) - } - } - } -} - -impl<'a> ITx for SledTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let tmp = self.save_error(tree.get(key))?; - Ok(tmp.map(|x| x.to_vec())) - } - fn len(&self, _tree: usize) -> TxOpResult { - unimplemented!(".len() in transaction not supported with Sled backend") - } - - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let old_val = self.save_error(tree.insert(key, value))?; - Ok(old_val.map(|x| x.to_vec())) - } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { - let tree = self.get_tree(tree)?; - let old_val = self.save_error(tree.remove(key))?; - Ok(old_val.map(|x| x.to_vec())) - } - - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - - fn range<'r>( - &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, - ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } - fn range_rev<'r>( - &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, - ) -> TxOpResult> { - unimplemented!("Iterators in transactions not supported with Sled backend"); - } -} diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 63b4506e..a03ee8ef 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -1,20 +1,35 @@ use core::ops::Bound; -use std::borrow::BorrowMut; use std::marker::PhantomPinned; +use std::path::PathBuf; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, RwLock}; -use rusqlite::{params, Connection, Rows, Statement, Transaction}; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::{params, Rows, Statement, Transaction}; use crate::{ - Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - TxValueIter, Value, ValueIter, + open::{Engine, OpenOpt}, + Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, + TxResult, TxValueIter, Value, ValueIter, }; pub use rusqlite; +// ---- top-level open function + +pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { + info!("Opening Sqlite database at: {}", path.display()); + let manager = r2d2_sqlite::SqliteConnectionManager::file(path); + Ok(SqliteDb::new(manager, opt.fsync)?) +} + +// ---- + +type Connection = r2d2::PooledConnection; + // --- err impl From for Error { @@ -23,6 +38,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: r2d2::Error) -> Error { + Error(format!("Sqlite: {}", e).into()) + } +} + impl From for TxOpError { fn from(e: rusqlite::Error) -> TxOpError { TxOpError(e.into()) @@ -31,35 +52,47 @@ impl From for TxOpError { // -- db -pub struct SqliteDb(Mutex); - -struct SqliteDbInner { - db: Connection, - trees: Vec, +pub struct SqliteDb { + db: Pool, + trees: RwLock>>, + // All operations that might write on the DB must take this lock first. + // This emulates LMDB's approach where a single writer can be + // active at once. + write_lock: Mutex<()>, } impl SqliteDb { - pub fn init(db: rusqlite::Connection) -> Db { - let s = Self(Mutex::new(SqliteDbInner { - db, - trees: Vec::new(), - })); - Db(Arc::new(s)) + pub fn new(manager: SqliteConnectionManager, sync_mode: bool) -> Result { + let manager = manager.with_init(move |db| { + db.pragma_update(None, "journal_mode", "WAL")?; + if sync_mode { + db.pragma_update(None, "synchronous", "NORMAL")?; + } else { + db.pragma_update(None, "synchronous", "OFF")?; + } + Ok(()) + }); + let s = Self { + db: Pool::builder().build(manager)?, + trees: RwLock::new(vec![]), + write_lock: Mutex::new(()), + }; + Ok(Db(Arc::new(s))) } } -impl SqliteDbInner { - fn get_tree(&self, i: usize) -> Result<&'_ str> { +impl SqliteDb { + fn get_tree(&self, i: usize) -> Result> { self.trees + .read() + .unwrap() .get(i) - .map(String::as_str) + .cloned() .ok_or_else(|| Error("invalid tree id".into())) } - fn internal_get(&self, tree: &str, key: &[u8]) -> Result> { - let mut stmt = self - .db - .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> Result> { + let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; let mut res_iter = stmt.query([key])?; match res_iter.next()? { None => Ok(None), @@ -75,13 +108,14 @@ impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { let name = format!("tree_{}", name.replace(':', "_COLON_")); - let mut this = self.0.lock().unwrap(); + let mut trees = self.trees.write().unwrap(); - if let Some(i) = this.trees.iter().position(|x| x == &name) { + if let Some(i) = trees.iter().position(|x| x.as_ref() == &name) { Ok(i) } else { + let db = self.db.get()?; trace!("create table {}", name); - this.db.execute( + db.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BLOB PRIMARY KEY, @@ -93,8 +127,8 @@ impl IDb for SqliteDb { )?; trace!("table created: {}, unlocking", name); - let i = this.trees.len(); - this.trees.push(name.to_string()); + let i = trees.len(); + trees.push(name.to_string().into_boxed_str().into()); Ok(i) } } @@ -102,11 +136,8 @@ impl IDb for SqliteDb { fn list_trees(&self) -> Result> { let mut trees = vec![]; - trace!("list_trees: lock db"); - let this = self.0.lock().unwrap(); - trace!("list_trees: lock acquired"); - - let mut stmt = this.db.prepare( + let db = self.db.get()?; + let mut stmt = db.prepare( "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", )?; let mut rows = stmt.query([])?; @@ -119,24 +150,33 @@ impl IDb for SqliteDb { Ok(trees) } + fn snapshot(&self, base_path: &PathBuf) -> Result<()> { + std::fs::create_dir_all(base_path)?; + let path = Engine::Sqlite + .db_path(&base_path) + .into_os_string() + .into_string() + .map_err(|_| Error("invalid sqlite path string".into()))?; + + info!("Start sqlite VACUUM INTO `{}`", path); + self.db.get()?.execute("VACUUM INTO ?1", params![path])?; + info!("Finished sqlite VACUUM INTO `{}`", path); + + Ok(()) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result> { - trace!("get {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("get {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; - this.internal_get(tree, key) + let tree = self.get_tree(tree)?; + self.internal_get(&self.db.get()?, &tree, key) } - fn len(&self, tree: usize) -> Result { - trace!("len {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("len {}: lock acquired", tree); + fn approximate_len(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; - let tree = this.get_tree(tree)?; - let mut stmt = this.db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; + let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; match res_iter.next()? { None => Ok(0), @@ -144,74 +184,60 @@ impl IDb for SqliteDb { } } - fn fast_len(&self, tree: usize) -> Result> { - Ok(Some(self.len(tree)?)) + fn is_empty(&self, tree: usize) -> Result { + Ok(self.approximate_len(tree)? == 0) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { - trace!("insert {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("insert {}: lock acquired", tree); + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - let old_val = this.internal_get(tree, key)?; + let old_val = self.internal_get(&db, &tree, key)?; let sql = match &old_val { Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), }; - let n = this.db.execute(&sql, params![key, value])?; + let n = db.execute(&sql, params![key, value])?; assert_eq!(n, 1); - Ok(old_val) + drop(lock); + Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result> { - trace!("remove {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("remove {}: lock acquired", tree); + fn remove(&self, tree: usize, key: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - let old_val = this.internal_get(tree, key)?; + db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - if old_val.is_some() { - let n = this - .db - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - assert_eq!(n, 1); - } - - Ok(old_val) + drop(lock); + Ok(()) } fn clear(&self, tree: usize) -> Result<()> { - trace!("clear {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("clear {}: lock acquired", tree); + let tree = self.get_tree(tree)?; + let db = self.db.get()?; + let lock = self.write_lock.lock(); - let tree = this.get_tree(tree)?; - this.db.execute(&format!("DELETE FROM {}", tree), [])?; + db.execute(&format!("DELETE FROM {}", tree), [])?; + + drop(lock); Ok(()) } fn iter(&self, tree: usize) -> Result> { - trace!("iter {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("iter {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - DbValueIterator::make(this, &sql, []) + DbValueIterator::make(self.db.get()?, &sql, []) } fn iter_rev(&self, tree: usize) -> Result> { - trace!("iter_rev {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("iter_rev {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); - DbValueIterator::make(this, &sql, []) + DbValueIterator::make(self.db.get()?, &sql, []) } fn range<'r>( @@ -220,11 +246,7 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - trace!("range {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("range {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); @@ -234,7 +256,7 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref()) } fn range_rev<'r>( &self, @@ -242,11 +264,7 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - trace!("range_rev {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("range_rev {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; + let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); @@ -256,30 +274,25 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref()) } // ---- - fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { - trace!("transaction: lock db"); - let mut this = self.0.lock().unwrap(); - trace!("transaction: lock acquired"); - - let this_mut_ref: &mut SqliteDbInner = this.borrow_mut(); + fn transaction(&self, f: &dyn ITxFn) -> TxResult { + let mut db = self.db.get().map_err(Error::from).map_err(TxError::Db)?; + let trees = self.trees.read().unwrap(); + let lock = self.write_lock.lock(); + trace!("trying transaction"); let mut tx = SqliteTx { - tx: this_mut_ref - .db - .transaction() - .map_err(Error::from) - .map_err(TxError::Db)?, - trees: &this_mut_ref.trees, + tx: db.transaction().map_err(Error::from).map_err(TxError::Db)?, + trees: &trees, }; let res = match f.try_on(&mut tx) { - TxFnResult::Ok => { + TxFnResult::Ok(on_commit) => { tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; - Ok(()) + Ok(on_commit) } TxFnResult::Abort => { tx.tx.rollback().map_err(Error::from).map_err(TxError::Db)?; @@ -294,7 +307,8 @@ impl IDb for SqliteDb { }; trace!("transaction done"); - res + drop(lock); + return res; } } @@ -302,14 +316,14 @@ impl IDb for SqliteDb { struct SqliteTx<'a> { tx: Transaction<'a>, - trees: &'a [String], + trees: &'a [Arc], } impl<'a> SqliteTx<'a> { fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> { - self.trees.get(i).map(String::as_ref).ok_or_else(|| { + self.trees.get(i).map(Arc::as_ref).ok_or_else(|| { TxOpError(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), + "invalid tree id (it might have been opened after the transaction started)".into(), )) }) } @@ -341,73 +355,86 @@ impl<'a> ITx for SqliteTx<'a> { } } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> { let tree = self.get_tree(tree)?; - let old_val = self.internal_get(tree, key)?; - - let sql = match &old_val { - Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), - None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), - }; - let n = self.tx.execute(&sql, params![key, value])?; - assert_eq!(n, 1); - - Ok(old_val) + let sql = format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree); + self.tx.execute(&sql, params![key, value])?; + Ok(()) } - fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> { let tree = self.get_tree(tree)?; - let old_val = self.internal_get(tree, key)?; - - if old_val.is_some() { - let n = self - .tx - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - assert_eq!(n, 1); - } - - Ok(old_val) + self.tx + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + Ok(()) + } + fn clear(&mut self, tree: usize) -> TxOpResult<()> { + let tree = self.get_tree(tree)?; + self.tx.execute(&format!("DELETE FROM {}", tree), [])?; + Ok(()) } - fn iter(&self, _tree: usize) -> TxOpResult> { - unimplemented!(); + fn iter(&self, tree: usize) -> TxOpResult> { + let tree = self.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); + TxValueIterator::make(self, &sql, []) } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { - unimplemented!(); + fn iter_rev(&self, tree: usize) -> TxOpResult> { + let tree = self.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); + TxValueIterator::make(self, &sql, []) } fn range<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!(); + let tree = self.get_tree(tree)?; + + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + + TxValueIterator::make::<&[&dyn rusqlite::ToSql]>(self, &sql, params.as_ref()) } fn range_rev<'r>( &self, - _tree: usize, - _low: Bound<&'r [u8]>, - _high: Bound<&'r [u8]>, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> TxOpResult> { - unimplemented!(); + let tree = self.get_tree(tree)?; + + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + + TxValueIterator::make::<&[&dyn rusqlite::ToSql]>(self, &sql, params.as_ref()) } } -// ---- +// ---- iterators outside transactions ---- +// complicated, they must hold the Statement and Row objects +// therefore quite some unsafe code (it is a self-referential struct) struct DbValueIterator<'a> { - db: MutexGuard<'a, SqliteDbInner>, + db: Connection, stmt: Option>, iter: Option>, _pin: PhantomPinned, } impl<'a> DbValueIterator<'a> { - fn make( - db: MutexGuard<'a, SqliteDbInner>, - sql: &str, - args: P, - ) -> Result> { + fn make(db: Connection, sql: &str, args: P) -> Result> { let res = DbValueIterator { db, stmt: None, @@ -417,17 +444,23 @@ impl<'a> DbValueIterator<'a> { let mut boxed = Box::pin(res); trace!("make iterator with sql: {}", sql); + // This unsafe allows us to bypass lifetime checks + let db = unsafe { NonNull::from(&boxed.db).as_ref() }; + let stmt = db.prepare(sql)?; + + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct unsafe { - let db = NonNull::from(&boxed.db); - let stmt = db.as_ref().db.prepare(sql)?; - - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); + } - let mut stmt = NonNull::from(&boxed.stmt); - let iter = stmt.as_mut().as_mut().unwrap().query(args)?; + // This unsafe allows us to bypass lifetime checks + let stmt = unsafe { NonNull::from(&boxed.stmt).as_mut() }; + let iter = stmt.as_mut().unwrap().query(args)?; - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct + unsafe { Pin::get_unchecked_mut(mut_ref).iter = Some(iter); } @@ -449,28 +482,73 @@ impl<'a> Iterator for DbValueIteratorPin<'a> { type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { - let next = unsafe { - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut self.0); - Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() - }; - let row = match next { - Err(e) => return Some(Err(e.into())), - Ok(None) => return None, - Ok(Some(r)) => r, - }; - let k = match row.get::<_, Vec>(0) { - Err(e) => return Some(Err(e.into())), - Ok(x) => x, - }; - let v = match row.get::<_, Vec>(1) { - Err(e) => return Some(Err(e.into())), - Ok(y) => y, - }; - Some(Ok((k, v))) + let mut_ref = Pin::as_mut(&mut self.0); + // This unsafe allows us to mutably access the iterator field + let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; + iter_next_row(next) } } -// ---- +// ---- iterators within transactions ---- +// it's the same except we don't hold a mutex guard, +// only a Statement and a Rows object + +struct TxValueIterator<'a> { + stmt: Statement<'a>, + iter: Option>, + _pin: PhantomPinned, +} + +impl<'a> TxValueIterator<'a> { + fn make( + tx: &'a SqliteTx<'a>, + sql: &str, + args: P, + ) -> TxOpResult> { + let stmt = tx.tx.prepare(sql)?; + let res = TxValueIterator { + stmt, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); + trace!("make iterator with sql: {}", sql); + + // This unsafe allows us to bypass lifetime checks + let stmt = unsafe { NonNull::from(&boxed.stmt).as_mut() }; + let iter = stmt.query(args)?; + + let mut_ref = Pin::as_mut(&mut boxed); + // This unsafe allows us to write in a field of the pinned struct + unsafe { + Pin::get_unchecked_mut(mut_ref).iter = Some(iter); + } + + Ok(Box::new(TxValueIteratorPin(boxed))) + } +} + +impl<'a> Drop for TxValueIterator<'a> { + fn drop(&mut self) { + trace!("drop iter"); + drop(self.iter.take()); + } +} + +struct TxValueIteratorPin<'a>(Pin>>); + +impl<'a> Iterator for TxValueIteratorPin<'a> { + type Item = TxOpResult<(Value, Value)>; + + fn next(&mut self) -> Option { + let mut_ref = Pin::as_mut(&mut self.0); + // This unsafe allows us to mutably access the iterator field + let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; + iter_next_row(next) + } +} + +// ---- utility ---- fn bounds_sql<'r>(low: Bound<&'r [u8]>, high: Bound<&'r [u8]>) -> (String, Vec>) { let mut sql = String::new(); @@ -510,3 +588,25 @@ fn bounds_sql<'r>(low: Bound<&'r [u8]>, high: Bound<&'r [u8]>) -> (String, Vec( + next_row: rusqlite::Result>, +) -> Option> +where + E: From, +{ + let row = match next_row { + Err(e) => return Some(Err(e.into())), + Ok(None) => return None, + Ok(Some(r)) => r, + }; + let k = match row.get::<_, Vec>(0) { + Err(e) => return Some(Err(e.into())), + Ok(x) => x, + }; + let v = match row.get::<_, Vec>(1) { + Err(e) => return Some(Err(e.into())), + Ok(y) => y, + }; + Some(Ok((k, v))) +} diff --git a/src/db/test.rs b/src/db/test.rs index 40e6c41e..977dc965 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -1,7 +1,7 @@ use crate::*; fn test_suite(db: Db) { - let tree = db.open_tree("tree").unwrap(); + let tree = db.open_tree("tree:this_is_a_tree").unwrap(); let ka: &[u8] = &b"test"[..]; let kb: &[u8] = &b"zwello"[..]; @@ -10,40 +10,47 @@ fn test_suite(db: Db) { let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - assert!(tree.insert(ka, va).unwrap().is_none()); - assert_eq!(tree.get(ka).unwrap().unwrap(), va); + // ---- test simple insert/delete ---- - let res = db.transaction::<_, (), _>(|mut tx| { + assert!(tree.insert(ka, va).is_ok()); + assert_eq!(tree.get(ka).unwrap().unwrap(), va); + assert_eq!(tree.iter().unwrap().count(), 1); + + // ---- test transaction logic ---- + + let res = db.transaction::<_, (), _>(|tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); - assert_eq!(tx.insert(&tree, ka, vb).unwrap().unwrap(), va); + assert_eq!(tx.insert(&tree, ka, vb).unwrap(), ()); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); - tx.commit(12) + Ok(12) }); assert!(matches!(res, Ok(12))); assert_eq!(tree.get(ka).unwrap().unwrap(), vb); - let res = db.transaction::<(), _, _>(|mut tx| { + let res = db.transaction::<(), _, _>(|tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); - assert_eq!(tx.insert(&tree, ka, vc).unwrap().unwrap(), vb); + assert_eq!(tx.insert(&tree, ka, vc).unwrap(), ()); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc); - tx.abort(42) + Err(TxError::Abort(42)) }); assert!(matches!(res, Err(TxError::Abort(42)))); assert_eq!(tree.get(ka).unwrap().unwrap(), vb); + // ---- test iteration outside of transactions ---- + let mut iter = tree.iter().unwrap(); let next = iter.next().unwrap().unwrap(); assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); - assert!(tree.insert(kb, vc).unwrap().is_none()); + assert!(tree.insert(kb, vc).is_ok()); assert_eq!(tree.get(kb).unwrap().unwrap(), vc); let mut iter = tree.iter().unwrap(); @@ -73,6 +80,48 @@ fn test_suite(db: Db) { assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); + + // ---- test iteration within transactions ---- + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.iter(&tree).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.range(&tree, kint..).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.range_rev(&tree, ..kint).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); + + db.transaction::<_, (), _>(|tx| { + let mut iter = tx.iter_rev(&tree).unwrap(); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + assert!(iter.next().is_none()); + Ok(()) + }) + .unwrap(); } #[test] @@ -90,22 +139,24 @@ fn test_lmdb_db() { drop(path); } -#[test] -#[cfg(feature = "sled")] -fn test_sled_db() { - use crate::sled_adapter::SledDb; - - let path = mktemp::Temp::new_dir().unwrap(); - let db = SledDb::init(sled::open(path.to_path_buf()).unwrap()); - test_suite(db); - drop(path); -} - #[test] #[cfg(feature = "sqlite")] fn test_sqlite_db() { use crate::sqlite_adapter::SqliteDb; - let db = SqliteDb::init(rusqlite::Connection::open_in_memory().unwrap()); + let manager = r2d2_sqlite::SqliteConnectionManager::memory(); + let db = SqliteDb::new(manager, false).unwrap(); + test_suite(db); +} + +#[test] +#[cfg(feature = "fjall")] +fn test_fjall_db() { + use crate::fjall_adapter::{fjall, FjallDb}; + + let path = mktemp::Temp::new_dir().unwrap(); + let config = fjall::Config::new(path).temporary(true); + let keyspace = config.open_transactional().unwrap(); + let db = FjallDb::init(keyspace); test_suite(db); } diff --git a/src/format-table/Cargo.toml b/src/format-table/Cargo.toml new file mode 100644 index 00000000..9e31e211 --- /dev/null +++ b/src/format-table/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "format_table" +version = "0.1.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Format tables with a stupid API" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "README.md" + +[lib] +path = "lib.rs" diff --git a/src/format-table/README.md b/src/format-table/README.md new file mode 100644 index 00000000..d918bdf4 --- /dev/null +++ b/src/format-table/README.md @@ -0,0 +1,13 @@ +# `format_table` + +Format tables with a stupid API. [Documentation](https://docs.rs/format_table). + +Example: + +```rust +let mut table = vec!["product\tquantity\tprice".to_string()]; +for (p, q, r) in [("tomato", 12, 15), ("potato", 10, 20), ("rice", 5, 12)] { + table.push(format!("{}\t{}\t{}", p, q, r)); +} +format_table::format_table(table); +``` diff --git a/src/util/formater.rs b/src/format-table/lib.rs similarity index 58% rename from src/util/formater.rs rename to src/format-table/lib.rs index 2ea53ebb..55252ba9 100644 --- a/src/util/formater.rs +++ b/src/format-table/lib.rs @@ -1,3 +1,19 @@ +//! Format tables with a stupid API. +//! +//! Example: +//! +//! ```rust +//! let mut table = vec!["product\tquantity\tprice".to_string()]; +//! for (p, q, r) in [("tomato", 12, 15), ("potato", 10, 20), ("rice", 5, 12)] { +//! table.push(format!("{}\t{}\t{}", p, q, r)); +//! } +//! format_table::format_table(table); +//! ``` +//! +//! A table to be formatted is a `Vec`, containing one string per line. +//! Table columns in each line are separated by a `\t` character. + +/// Format a table and return the result as a string. pub fn format_table_to_string(data: Vec) -> String { let data = data .iter() @@ -27,6 +43,7 @@ pub fn format_table_to_string(data: Vec) -> String { out } +/// Format a table and print the result to stdout. pub fn format_table(data: Vec) { print!("{}", format_table_to_string(data)); } diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 0cbdf890..a4f695a4 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -21,75 +21,90 @@ path = "tests/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.2", path = "../db" } -garage_api = { version = "0.8.2", path = "../api" } -garage_block = { version = "0.8.2", path = "../block" } -garage_model = { version = "0.8.2", path = "../model" } -garage_rpc = { version = "0.8.2", path = "../rpc" } -garage_table = { version = "0.8.2", path = "../table" } -garage_util = { version = "0.8.2", path = "../util" } -garage_web = { version = "0.8.2", path = "../web" } +format_table.workspace = true +garage_db.workspace = true +garage_api_admin.workspace = true +garage_api_s3.workspace = true +garage_api_k2v = { workspace = true, optional = true } +garage_block.workspace = true +garage_model.workspace = true +garage_net.workspace = true +garage_rpc.workspace = true +garage_table.workspace = true +garage_util.workspace = true +garage_web.workspace = true -backtrace = "0.3" -bytes = "1.0" -bytesize = "1.1" -timeago = "0.4" -parse_duration = "2.1" -hex = "0.4" -tracing = { version = "0.1" } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -rand = "0.8" -async-trait = "0.1.7" -sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } +backtrace.workspace = true +bytes.workspace = true +bytesize.workspace = true +timeago.workspace = true +parse_duration.workspace = true +hex.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +async-trait.workspace = true +sha1.workspace = true +sodiumoxide.workspace = true +structopt.workspace = true +git-version.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -structopt = { version = "0.3", default-features = false } -toml = "0.6" +serde.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +tokio.workspace = true -netapp = "0.5" - -opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } -opentelemetry-prometheus = { version = "0.10", optional = true } -opentelemetry-otlp = { version = "0.10", optional = true } -prometheus = { version = "0.13", optional = true } +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true } +syslog-tracing = { workspace = true, optional = true } +tracing-journald = { workspace = true, optional = true } [dev-dependencies] -aws-sdk-s3 = "0.19" -chrono = "0.4" -http = "0.2" -hmac = "0.12" -hyper = { version = "0.14", features = ["client", "http1", "runtime"] } -sha2 = "0.10" +garage_api_common.workspace = true -static_init = "1.0" -assert-json-diff = "2.0" -serde_json = "1.0" -base64 = "0.21" +aws-sdk-s3.workspace = true +aws-smithy-runtime.workspace = true +chrono.workspace = true +http.workspace = true +hmac.workspace = true +http-body-util.workspace = true +hyper.workspace = true +hyper-util.workspace = true +mktemp.workspace = true +sha2.workspace = true + + +static_init.workspace = true +assert-json-diff.workspace = true +serde_json.workspace = true +base64.workspace = true +crc32fast.workspace = true + +k2v-client.workspace = true [features] -default = [ "bundled-libs", "metrics", "sled", "k2v" ] +default = [ "bundled-libs", "metrics", "lmdb", "sqlite", "k2v" ] -k2v = [ "garage_util/k2v", "garage_api/k2v" ] +k2v = [ "garage_util/k2v", "garage_api_k2v" ] -# Database engines, Sled is still our default even though we don't like it -sled = [ "garage_model/sled" ] +# Database engines lmdb = [ "garage_model/lmdb" ] sqlite = [ "garage_model/sqlite" ] +fjall = [ "garage_model/fjall" ] # Automatic registration and discovery via Consul API consul-discovery = [ "garage_rpc/consul-discovery" ] # Automatic registration and discovery via Kubernetes API kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] # Prometheus exporter (/metrics endpoint). -metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ] +metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ] # Exporter for the OpenTelemetry Collector. telemetry-otlp = [ "opentelemetry-otlp" ] +# Logging to syslog +syslog = [ "syslog-tracing" ] +# Logging to journald +journald = [ "tracing-journald" ] # NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # exactly one of them should be enabled. diff --git a/src/garage/admin.rs b/src/garage/admin.rs deleted file mode 100644 index 34141cb2..00000000 --- a/src/garage/admin.rs +++ /dev/null @@ -1,1298 +0,0 @@ -use std::collections::HashMap; -use std::fmt::Write; -use std::sync::Arc; - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -use garage_util::background::BackgroundRunner; -use garage_util::crdt::*; -use garage_util::data::*; -use garage_util::error::Error as GarageError; -use garage_util::formater::format_table_to_string; -use garage_util::time::*; - -use garage_table::replication::*; -use garage_table::*; - -use garage_rpc::ring::PARTITION_BITS; -use garage_rpc::*; - -use garage_block::manager::BlockResyncErrorInfo; - -use garage_model::bucket_alias_table::*; -use garage_model::bucket_table::*; -use garage_model::garage::Garage; -use garage_model::helper::error::{Error, OkOrBadRequest}; -use garage_model::key_table::*; -use garage_model::migrate::Migrate; -use garage_model::permission::*; -use garage_model::s3::object_table::*; -use garage_model::s3::version_table::Version; - -use crate::cli::*; -use crate::repair::online::launch_online_repair; - -pub const ADMIN_RPC_PATH: &str = "garage/admin_rpc.rs/Rpc"; - -#[derive(Debug, Serialize, Deserialize)] -#[allow(clippy::large_enum_variant)] -pub enum AdminRpc { - BucketOperation(BucketOperation), - KeyOperation(KeyOperation), - LaunchRepair(RepairOpt), - Migrate(MigrateOpt), - Stats(StatsOpt), - Worker(WorkerOperation), - BlockOperation(BlockOperation), - - // Replies - Ok(String), - BucketList(Vec), - BucketInfo { - bucket: Bucket, - relevant_keys: HashMap, - counters: HashMap, - }, - KeyList(Vec<(String, String)>), - KeyInfo(Key, HashMap), - WorkerList( - HashMap, - WorkerListOpt, - ), - WorkerVars(Vec<(Uuid, String, String)>), - WorkerInfo(usize, garage_util::background::WorkerInfo), - BlockErrorList(Vec), - BlockInfo { - hash: Hash, - refcount: u64, - versions: Vec>, - }, -} - -impl Rpc for AdminRpc { - type Response = Result; -} - -pub struct AdminRpcHandler { - garage: Arc, - background: Arc, - endpoint: Arc>, -} - -impl AdminRpcHandler { - pub fn new(garage: Arc, background: Arc) -> Arc { - let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into()); - let admin = Arc::new(Self { - garage, - background, - endpoint, - }); - admin.endpoint.set_handler(admin.clone()); - admin - } - - // ================ BUCKET COMMANDS ==================== - - async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result { - match cmd { - BucketOperation::List => self.handle_list_buckets().await, - BucketOperation::Info(query) => self.handle_bucket_info(query).await, - BucketOperation::Create(query) => self.handle_create_bucket(&query.name).await, - BucketOperation::Delete(query) => self.handle_delete_bucket(query).await, - BucketOperation::Alias(query) => self.handle_alias_bucket(query).await, - BucketOperation::Unalias(query) => self.handle_unalias_bucket(query).await, - BucketOperation::Allow(query) => self.handle_bucket_allow(query).await, - BucketOperation::Deny(query) => self.handle_bucket_deny(query).await, - BucketOperation::Website(query) => self.handle_bucket_website(query).await, - BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await, - BucketOperation::CleanupIncompleteUploads(query) => { - self.handle_bucket_cleanup_incomplete_uploads(query).await - } - } - } - - async fn handle_list_buckets(&self) -> Result { - let buckets = self - .garage - .bucket_table - .get_range( - &EmptyKey, - None, - Some(DeletedFilter::NotDeleted), - 10000, - EnumerationOrder::Forward, - ) - .await?; - - Ok(AdminRpc::BucketList(buckets)) - } - - async fn handle_bucket_info(&self, query: &BucketOpt) -> Result { - let bucket_id = self - .garage - .bucket_helper() - .resolve_global_bucket_name(&query.name) - .await? - .ok_or_bad_request("Bucket not found")?; - - let bucket = self - .garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - - let counters = self - .garage - .object_counter_table - .table - .get(&bucket_id, &EmptyKey) - .await? - .map(|x| x.filtered_values(&self.garage.system.ring.borrow())) - .unwrap_or_default(); - - let mut relevant_keys = HashMap::new(); - for (k, _) in bucket - .state - .as_option() - .unwrap() - .authorized_keys - .items() - .iter() - { - if let Some(key) = self - .garage - .key_table - .get(&EmptyKey, k) - .await? - .filter(|k| !k.is_deleted()) - { - relevant_keys.insert(k.clone(), key); - } - } - for ((k, _), _, _) in bucket - .state - .as_option() - .unwrap() - .local_aliases - .items() - .iter() - { - if relevant_keys.contains_key(k) { - continue; - } - if let Some(key) = self.garage.key_table.get(&EmptyKey, k).await? { - relevant_keys.insert(k.clone(), key); - } - } - - Ok(AdminRpc::BucketInfo { - bucket, - relevant_keys, - counters, - }) - } - - #[allow(clippy::ptr_arg)] - async fn handle_create_bucket(&self, name: &String) -> Result { - if !is_valid_bucket_name(name) { - return Err(Error::BadRequest(format!( - "{}: {}", - name, INVALID_BUCKET_NAME_MESSAGE - ))); - } - - if let Some(alias) = self.garage.bucket_alias_table.get(&EmptyKey, name).await? { - if alias.state.get().is_some() { - return Err(Error::BadRequest(format!("Bucket {} already exists", name))); - } - } - - // ---- done checking, now commit ---- - - let bucket = Bucket::new(); - self.garage.bucket_table.insert(&bucket).await?; - - self.garage - .bucket_helper() - .set_global_bucket_alias(bucket.id, name) - .await?; - - Ok(AdminRpc::Ok(format!("Bucket {} was created.", name))) - } - - async fn handle_delete_bucket(&self, query: &DeleteBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - - let bucket_id = helper - .resolve_global_bucket_name(&query.name) - .await? - .ok_or_bad_request("Bucket not found")?; - - // Get the alias, but keep in minde here the bucket name - // given in parameter can also be directly the bucket's ID. - // In that case bucket_alias will be None, and - // we can still delete the bucket if it has zero aliases - // (a condition which we try to prevent but that could still happen somehow). - // We just won't try to delete an alias entry because there isn't one. - let bucket_alias = self - .garage - .bucket_alias_table - .get(&EmptyKey, &query.name) - .await?; - - // Check bucket doesn't have other aliases - let mut bucket = helper.get_existing_bucket(bucket_id).await?; - let bucket_state = bucket.state.as_option().unwrap(); - if bucket_state - .aliases - .items() - .iter() - .filter(|(_, _, active)| *active) - .any(|(name, _, _)| name != &query.name) - { - return Err(Error::BadRequest(format!("Bucket {} still has other global aliases. Use `bucket unalias` to delete them one by one.", query.name))); - } - if bucket_state - .local_aliases - .items() - .iter() - .any(|(_, _, active)| *active) - { - return Err(Error::BadRequest(format!("Bucket {} still has other local aliases. Use `bucket unalias` to delete them one by one.", query.name))); - } - - // Check bucket is empty - if !helper.is_bucket_empty(bucket_id).await? { - return Err(Error::BadRequest(format!( - "Bucket {} is not empty", - query.name - ))); - } - - if !query.yes { - return Err(Error::BadRequest( - "Add --yes flag to really perform this operation".to_string(), - )); - } - - // --- done checking, now commit --- - // 1. delete authorization from keys that had access - for (key_id, _) in bucket.authorized_keys() { - helper - .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) - .await?; - } - - // 2. delete bucket alias - if bucket_alias.is_some() { - helper - .purge_global_bucket_alias(bucket_id, &query.name) - .await?; - } - - // 3. delete bucket - bucket.state = Deletable::delete(); - self.garage.bucket_table.insert(&bucket).await?; - - Ok(AdminRpc::Ok(format!("Bucket {} was deleted.", query.name))) - } - - async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); - - let bucket_id = helper - .resolve_global_bucket_name(&query.existing_bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - - if let Some(key_pattern) = &query.local { - let key = key_helper.get_existing_matching_key(key_pattern).await?; - - helper - .set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name) - .await?; - Ok(AdminRpc::Ok(format!( - "Alias {} now points to bucket {:?} in namespace of key {}", - query.new_name, bucket_id, key.key_id - ))) - } else { - helper - .set_global_bucket_alias(bucket_id, &query.new_name) - .await?; - Ok(AdminRpc::Ok(format!( - "Alias {} now points to bucket {:?}", - query.new_name, bucket_id - ))) - } - } - - async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); - - if let Some(key_pattern) = &query.local { - let key = key_helper.get_existing_matching_key(key_pattern).await?; - - let bucket_id = key - .state - .as_option() - .unwrap() - .local_aliases - .get(&query.name) - .cloned() - .flatten() - .ok_or_bad_request("Bucket not found")?; - - helper - .unset_local_bucket_alias(bucket_id, &key.key_id, &query.name) - .await?; - - Ok(AdminRpc::Ok(format!( - "Alias {} no longer points to bucket {:?} in namespace of key {}", - &query.name, bucket_id, key.key_id - ))) - } else { - let bucket_id = helper - .resolve_global_bucket_name(&query.name) - .await? - .ok_or_bad_request("Bucket not found")?; - - helper - .unset_global_bucket_alias(bucket_id, &query.name) - .await?; - - Ok(AdminRpc::Ok(format!( - "Alias {} no longer points to bucket {:?}", - &query.name, bucket_id - ))) - } - } - - async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); - - let bucket_id = helper - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - let key = key_helper - .get_existing_matching_key(&query.key_pattern) - .await?; - - let allow_read = query.read || key.allow_read(&bucket_id); - let allow_write = query.write || key.allow_write(&bucket_id); - let allow_owner = query.owner || key.allow_owner(&bucket_id); - - helper - .set_bucket_key_permissions( - bucket_id, - &key.key_id, - BucketKeyPerm { - timestamp: now_msec(), - allow_read, - allow_write, - allow_owner, - }, - ) - .await?; - - Ok(AdminRpc::Ok(format!( - "New permissions for {} on {}: read {}, write {}, owner {}.", - &key.key_id, &query.bucket, allow_read, allow_write, allow_owner - ))) - } - - async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result { - let helper = self.garage.bucket_helper(); - let key_helper = self.garage.key_helper(); - - let bucket_id = helper - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - let key = key_helper - .get_existing_matching_key(&query.key_pattern) - .await?; - - let allow_read = !query.read && key.allow_read(&bucket_id); - let allow_write = !query.write && key.allow_write(&bucket_id); - let allow_owner = !query.owner && key.allow_owner(&bucket_id); - - helper - .set_bucket_key_permissions( - bucket_id, - &key.key_id, - BucketKeyPerm { - timestamp: now_msec(), - allow_read, - allow_write, - allow_owner, - }, - ) - .await?; - - Ok(AdminRpc::Ok(format!( - "New permissions for {} on {}: read {}, write {}, owner {}.", - &key.key_id, &query.bucket, allow_read, allow_write, allow_owner - ))) - } - - async fn handle_bucket_website(&self, query: &WebsiteOpt) -> Result { - let bucket_id = self - .garage - .bucket_helper() - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - - let mut bucket = self - .garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - let bucket_state = bucket.state.as_option_mut().unwrap(); - - if !(query.allow ^ query.deny) { - return Err(Error::BadRequest( - "You must specify exactly one flag, either --allow or --deny".to_string(), - )); - } - - let website = if query.allow { - Some(WebsiteConfig { - index_document: query.index_document.clone(), - error_document: query.error_document.clone(), - }) - } else { - None - }; - - bucket_state.website_config.update(website); - self.garage.bucket_table.insert(&bucket).await?; - - let msg = if query.allow { - format!("Website access allowed for {}", &query.bucket) - } else { - format!("Website access denied for {}", &query.bucket) - }; - - Ok(AdminRpc::Ok(msg)) - } - - async fn handle_bucket_set_quotas(&self, query: &SetQuotasOpt) -> Result { - let bucket_id = self - .garage - .bucket_helper() - .resolve_global_bucket_name(&query.bucket) - .await? - .ok_or_bad_request("Bucket not found")?; - - let mut bucket = self - .garage - .bucket_helper() - .get_existing_bucket(bucket_id) - .await?; - let bucket_state = bucket.state.as_option_mut().unwrap(); - - if query.max_size.is_none() && query.max_objects.is_none() { - return Err(Error::BadRequest( - "You must specify either --max-size or --max-objects (or both) for this command to do something.".to_string(), - )); - } - - let mut quotas = bucket_state.quotas.get().clone(); - - match query.max_size.as_ref().map(String::as_ref) { - Some("none") => quotas.max_size = None, - Some(v) => { - let bs = v - .parse::() - .ok_or_bad_request(format!("Invalid size specified: {}", v))?; - quotas.max_size = Some(bs.as_u64()); - } - _ => (), - } - - match query.max_objects.as_ref().map(String::as_ref) { - Some("none") => quotas.max_objects = None, - Some(v) => { - let mo = v - .parse::() - .ok_or_bad_request(format!("Invalid number specified: {}", v))?; - quotas.max_objects = Some(mo); - } - _ => (), - } - - bucket_state.quotas.update(quotas); - self.garage.bucket_table.insert(&bucket).await?; - - Ok(AdminRpc::Ok(format!( - "Quotas updated for {}", - &query.bucket - ))) - } - - async fn handle_bucket_cleanup_incomplete_uploads( - &self, - query: &CleanupIncompleteUploadsOpt, - ) -> Result { - let mut bucket_ids = vec![]; - for b in query.buckets.iter() { - bucket_ids.push( - self.garage - .bucket_helper() - .resolve_global_bucket_name(b) - .await? - .ok_or_bad_request(format!("Bucket not found: {}", b))?, - ); - } - - let duration = parse_duration::parse::parse(&query.older_than) - .ok_or_bad_request("Invalid duration passed for --older-than parameter")?; - - let mut ret = String::new(); - for bucket in bucket_ids { - let count = self - .garage - .bucket_helper() - .cleanup_incomplete_uploads(&bucket, duration) - .await?; - writeln!( - &mut ret, - "Bucket {:?}: {} incomplete uploads aborted", - bucket, count - ) - .unwrap(); - } - - Ok(AdminRpc::Ok(ret)) - } - - // ================ KEY COMMANDS ==================== - - async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result { - match cmd { - KeyOperation::List => self.handle_list_keys().await, - KeyOperation::Info(query) => self.handle_key_info(query).await, - KeyOperation::New(query) => self.handle_create_key(query).await, - KeyOperation::Rename(query) => self.handle_rename_key(query).await, - KeyOperation::Delete(query) => self.handle_delete_key(query).await, - KeyOperation::Allow(query) => self.handle_allow_key(query).await, - KeyOperation::Deny(query) => self.handle_deny_key(query).await, - KeyOperation::Import(query) => self.handle_import_key(query).await, - } - } - - async fn handle_list_keys(&self) -> Result { - let key_ids = self - .garage - .key_table - .get_range( - &EmptyKey, - None, - Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)), - 10000, - EnumerationOrder::Forward, - ) - .await? - .iter() - .map(|k| (k.key_id.to_string(), k.params().unwrap().name.get().clone())) - .collect::>(); - Ok(AdminRpc::KeyList(key_ids)) - } - - async fn handle_key_info(&self, query: &KeyOpt) -> Result { - let key = self - .garage - .key_helper() - .get_existing_matching_key(&query.key_pattern) - .await?; - self.key_info_result(key).await - } - - async fn handle_create_key(&self, query: &KeyNewOpt) -> Result { - let key = Key::new(&query.name); - self.garage.key_table.insert(&key).await?; - self.key_info_result(key).await - } - - async fn handle_rename_key(&self, query: &KeyRenameOpt) -> Result { - let mut key = self - .garage - .key_helper() - .get_existing_matching_key(&query.key_pattern) - .await?; - key.params_mut() - .unwrap() - .name - .update(query.new_name.clone()); - self.garage.key_table.insert(&key).await?; - self.key_info_result(key).await - } - - async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result { - let key_helper = self.garage.key_helper(); - - let mut key = key_helper - .get_existing_matching_key(&query.key_pattern) - .await?; - - if !query.yes { - return Err(Error::BadRequest( - "Add --yes flag to really perform this operation".to_string(), - )); - } - - key_helper.delete_key(&mut key).await?; - - Ok(AdminRpc::Ok(format!( - "Key {} was deleted successfully.", - key.key_id - ))) - } - - async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result { - let mut key = self - .garage - .key_helper() - .get_existing_matching_key(&query.key_pattern) - .await?; - if query.create_bucket { - key.params_mut().unwrap().allow_create_bucket.update(true); - } - self.garage.key_table.insert(&key).await?; - self.key_info_result(key).await - } - - async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result { - let mut key = self - .garage - .key_helper() - .get_existing_matching_key(&query.key_pattern) - .await?; - if query.create_bucket { - key.params_mut().unwrap().allow_create_bucket.update(false); - } - self.garage.key_table.insert(&key).await?; - self.key_info_result(key).await - } - - async fn handle_import_key(&self, query: &KeyImportOpt) -> Result { - let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?; - if prev_key.is_some() { - return Err(Error::BadRequest(format!("Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", query.key_id))); - } - let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name); - self.garage.key_table.insert(&imported_key).await?; - - self.key_info_result(imported_key).await - } - - async fn key_info_result(&self, key: Key) -> Result { - let mut relevant_buckets = HashMap::new(); - - for (id, _) in key - .state - .as_option() - .unwrap() - .authorized_buckets - .items() - .iter() - { - if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? { - relevant_buckets.insert(*id, b); - } - } - - Ok(AdminRpc::KeyInfo(key, relevant_buckets)) - } - - // ================ MIGRATION COMMANDS ==================== - - async fn handle_migrate(self: &Arc, opt: MigrateOpt) -> Result { - if !opt.yes { - return Err(Error::BadRequest( - "Please provide the --yes flag to initiate migration operation.".to_string(), - )); - } - - let m = Migrate { - garage: self.garage.clone(), - }; - match opt.what { - MigrateWhat::Buckets050 => m.migrate_buckets050().await, - }?; - Ok(AdminRpc::Ok("Migration successfull.".into())) - } - - // ================ REPAIR COMMANDS ==================== - - async fn handle_launch_repair(self: &Arc, opt: RepairOpt) -> Result { - if !opt.yes { - return Err(Error::BadRequest( - "Please provide the --yes flag to initiate repair operations.".to_string(), - )); - } - if opt.all_nodes { - let mut opt_to_send = opt.clone(); - opt_to_send.all_nodes = false; - - let mut failures = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { - let node = (*node).into(); - let resp = self - .endpoint - .call( - &node, - AdminRpc::LaunchRepair(opt_to_send.clone()), - PRIO_NORMAL, - ) - .await; - if !matches!(resp, Ok(Ok(_))) { - failures.push(node); - } - } - if failures.is_empty() { - Ok(AdminRpc::Ok("Repair launched on all nodes".to_string())) - } else { - Err(Error::BadRequest(format!( - "Could not launch repair on nodes: {:?} (launched successfully on other nodes)", - failures - ))) - } - } else { - launch_online_repair(&self.garage, &self.background, opt).await?; - Ok(AdminRpc::Ok(format!( - "Repair launched on {:?}", - self.garage.system.id - ))) - } - } - - // ================ STATS COMMANDS ==================== - - async fn handle_stats(&self, opt: StatsOpt) -> Result { - if opt.all_nodes { - let mut ret = String::new(); - let ring = self.garage.system.ring.borrow().clone(); - - for node in ring.layout.node_ids().iter() { - let mut opt = opt.clone(); - opt.all_nodes = false; - opt.skip_global = true; - - writeln!(&mut ret, "\n======================").unwrap(); - writeln!(&mut ret, "Stats for node {:?}:", node).unwrap(); - - let node_id = (*node).into(); - match self - .endpoint - .call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL) - .await - { - Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(), - Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(), - Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(), - Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(), - } - } - - writeln!(&mut ret, "\n======================").unwrap(); - write!( - &mut ret, - "Cluster statistics:\n\n{}", - self.gather_cluster_stats() - ) - .unwrap(); - - Ok(AdminRpc::Ok(ret)) - } else { - Ok(AdminRpc::Ok(self.gather_stats_local(opt)?)) - } - } - - fn gather_stats_local(&self, opt: StatsOpt) -> Result { - let mut ret = String::new(); - writeln!( - &mut ret, - "\nGarage version: {} [features: {}]\nRust compiler version: {}", - garage_util::version::garage_version(), - garage_util::version::garage_features() - .map(|list| list.join(", ")) - .unwrap_or_else(|| "(unknown)".into()), - garage_util::version::rust_version(), - ) - .unwrap(); - - writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap(); - - // Gather table statistics - let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tGcTodo".into()]; - table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?); - table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?); - write!( - &mut ret, - "\nTable stats:\n{}", - format_table_to_string(table) - ) - .unwrap(); - - // Gather block manager statistics - writeln!(&mut ret, "\nBlock manager stats:").unwrap(); - let rc_len = if opt.detailed { - self.garage.block_manager.rc_len()?.to_string() - } else { - self.garage - .block_manager - .rc_fast_len()? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()) - }; - - writeln!( - &mut ret, - " number of RC entries (~= number of blocks): {}", - rc_len - ) - .unwrap(); - writeln!( - &mut ret, - " resync queue length: {}", - self.garage.block_manager.resync.queue_len()? - ) - .unwrap(); - writeln!( - &mut ret, - " blocks with resync errors: {}", - self.garage.block_manager.resync.errors_len()? - ) - .unwrap(); - - if !opt.detailed { - writeln!(&mut ret, "\nIf values are missing above (marked as NC), consider adding the --detailed flag (this will be slow).").unwrap(); - } - - if !opt.skip_global { - write!(&mut ret, "\n{}", self.gather_cluster_stats()).unwrap(); - } - - Ok(ret) - } - - fn gather_cluster_stats(&self) -> String { - let mut ret = String::new(); - - // Gather storage node and free space statistics - let layout = &self.garage.system.ring.borrow().layout; - let mut node_partition_count = HashMap::::new(); - for short_id in layout.ring_assignation_data.iter() { - let id = layout.node_id_vec[*short_id as usize]; - *node_partition_count.entry(id).or_default() += 1; - } - let node_info = self - .garage - .system - .get_known_nodes() - .into_iter() - .map(|n| (n.id, n)) - .collect::>(); - - let mut table = vec![" ID\tHostname\tZone\tCapacity\tPart.\tDataAvail\tMetaAvail".into()]; - for (id, parts) in node_partition_count.iter() { - let info = node_info.get(id); - let status = info.map(|x| &x.status); - let role = layout.roles.get(id).and_then(|x| x.0.as_ref()); - let hostname = status.map(|x| x.hostname.as_str()).unwrap_or("?"); - let zone = role.map(|x| x.zone.as_str()).unwrap_or("?"); - let capacity = role - .map(|x| x.capacity_string()) - .unwrap_or_else(|| "?".into()); - let avail_str = |x| match x { - Some((avail, total)) => { - let pct = (avail as f64) / (total as f64) * 100.; - let avail = bytesize::ByteSize::b(avail); - let total = bytesize::ByteSize::b(total); - format!("{}/{} ({:.1}%)", avail, total, pct) - } - None => "?".into(), - }; - let data_avail = avail_str(status.and_then(|x| x.data_disk_avail)); - let meta_avail = avail_str(status.and_then(|x| x.meta_disk_avail)); - table.push(format!( - " {:?}\t{}\t{}\t{}\t{}\t{}\t{}", - id, hostname, zone, capacity, parts, data_avail, meta_avail - )); - } - write!( - &mut ret, - "Storage nodes:\n{}", - format_table_to_string(table) - ) - .unwrap(); - - let meta_part_avail = node_partition_count - .iter() - .filter_map(|(id, parts)| { - node_info - .get(id) - .and_then(|x| x.status.meta_disk_avail) - .map(|c| c.0 / *parts) - }) - .collect::>(); - let data_part_avail = node_partition_count - .iter() - .filter_map(|(id, parts)| { - node_info - .get(id) - .and_then(|x| x.status.data_disk_avail) - .map(|c| c.0 / *parts) - }) - .collect::>(); - if !meta_part_avail.is_empty() && !data_part_avail.is_empty() { - let meta_avail = - bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); - let data_avail = - bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); - writeln!( - &mut ret, - "\nEstimated available storage space cluster-wide (might be lower in practice):" - ) - .unwrap(); - if meta_part_avail.len() < node_partition_count.len() - || data_part_avail.len() < node_partition_count.len() - { - writeln!(&mut ret, " data: < {}", data_avail).unwrap(); - writeln!(&mut ret, " metadata: < {}", meta_avail).unwrap(); - writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap(); - } else { - writeln!(&mut ret, " data: {}", data_avail).unwrap(); - writeln!(&mut ret, " metadata: {}", meta_avail).unwrap(); - } - } - - ret - } - - fn gather_table_stats( - &self, - t: &Arc>, - detailed: bool, - ) -> Result - where - F: TableSchema + 'static, - R: TableReplication + 'static, - { - let (data_len, mkl_len) = if detailed { - ( - t.data.store.len().map_err(GarageError::from)?.to_string(), - t.merkle_updater.merkle_tree_len()?.to_string(), - ) - } else { - ( - t.data - .store - .fast_len() - .map_err(GarageError::from)? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()), - t.merkle_updater - .merkle_tree_fast_len()? - .map(|x| x.to_string()) - .unwrap_or_else(|| "NC".into()), - ) - }; - - Ok(format!( - " {}\t{}\t{}\t{}\t{}", - F::TABLE_NAME, - data_len, - mkl_len, - t.merkle_updater.todo_len()?, - t.data.gc_todo_len()? - )) - } - - // ================ WORKER COMMANDS ==================== - - async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result { - match cmd { - WorkerOperation::List { opt } => { - let workers = self.background.get_worker_info(); - Ok(AdminRpc::WorkerList(workers, *opt)) - } - WorkerOperation::Info { tid } => { - let info = self - .background - .get_worker_info() - .get(tid) - .ok_or_bad_request(format!("No worker with TID {}", tid))? - .clone(); - Ok(AdminRpc::WorkerInfo(*tid, info)) - } - WorkerOperation::Get { - all_nodes, - variable, - } => self.handle_get_var(*all_nodes, variable).await, - WorkerOperation::Set { - all_nodes, - variable, - value, - } => self.handle_set_var(*all_nodes, variable, value).await, - } - } - - async fn handle_get_var( - &self, - all_nodes: bool, - variable: &Option, - ) -> Result { - if all_nodes { - let mut ret = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { - let node = (*node).into(); - match self - .endpoint - .call( - &node, - AdminRpc::Worker(WorkerOperation::Get { - all_nodes: false, - variable: variable.clone(), - }), - PRIO_NORMAL, - ) - .await?? - { - AdminRpc::WorkerVars(v) => ret.extend(v), - m => return Err(GarageError::unexpected_rpc_message(m).into()), - } - } - Ok(AdminRpc::WorkerVars(ret)) - } else { - #[allow(clippy::collapsible_else_if)] - if let Some(v) = variable { - Ok(AdminRpc::WorkerVars(vec![( - self.garage.system.id, - v.clone(), - self.garage.bg_vars.get(v)?, - )])) - } else { - let mut vars = self.garage.bg_vars.get_all(); - vars.sort(); - Ok(AdminRpc::WorkerVars( - vars.into_iter() - .map(|(k, v)| (self.garage.system.id, k.to_string(), v)) - .collect(), - )) - } - } - } - - async fn handle_set_var( - &self, - all_nodes: bool, - variable: &str, - value: &str, - ) -> Result { - if all_nodes { - let mut ret = vec![]; - let ring = self.garage.system.ring.borrow().clone(); - for node in ring.layout.node_ids().iter() { - let node = (*node).into(); - match self - .endpoint - .call( - &node, - AdminRpc::Worker(WorkerOperation::Set { - all_nodes: false, - variable: variable.to_string(), - value: value.to_string(), - }), - PRIO_NORMAL, - ) - .await?? - { - AdminRpc::WorkerVars(v) => ret.extend(v), - m => return Err(GarageError::unexpected_rpc_message(m).into()), - } - } - Ok(AdminRpc::WorkerVars(ret)) - } else { - self.garage.bg_vars.set(variable, value)?; - Ok(AdminRpc::WorkerVars(vec![( - self.garage.system.id, - variable.to_string(), - value.to_string(), - )])) - } - } - - // ================ BLOCK COMMANDS ==================== - - async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result { - match cmd { - BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList( - self.garage.block_manager.list_resync_errors()?, - )), - BlockOperation::Info { hash } => { - let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; - let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; - let refcount = self.garage.block_manager.get_block_rc(&hash)?; - let block_refs = self - .garage - .block_ref_table - .get_range(&hash, None, None, 10000, Default::default()) - .await?; - let mut versions = vec![]; - for br in block_refs { - if let Some(v) = self - .garage - .version_table - .get(&br.version, &EmptyKey) - .await? - { - versions.push(Ok(v)); - } else { - versions.push(Err(br.version)); - } - } - Ok(AdminRpc::BlockInfo { - hash, - refcount, - versions, - }) - } - BlockOperation::RetryNow { all, blocks } => { - if *all { - if !blocks.is_empty() { - return Err(Error::BadRequest( - "--all was specified, cannot also specify blocks".into(), - )); - } - let blocks = self.garage.block_manager.list_resync_errors()?; - for b in blocks.iter() { - self.garage.block_manager.resync.clear_backoff(&b.hash)?; - } - Ok(AdminRpc::Ok(format!( - "{} blocks returned in queue for a retry now (check logs to see results)", - blocks.len() - ))) - } else { - for hash in blocks { - let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; - let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; - self.garage.block_manager.resync.clear_backoff(&hash)?; - } - Ok(AdminRpc::Ok(format!( - "{} blocks returned in queue for a retry now (check logs to see results)", - blocks.len() - ))) - } - } - BlockOperation::Purge { yes, blocks } => { - if !yes { - return Err(Error::BadRequest( - "Pass the --yes flag to confirm block purge operation.".into(), - )); - } - - let mut obj_dels = 0; - let mut ver_dels = 0; - - for hash in blocks { - let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; - let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; - let block_refs = self - .garage - .block_ref_table - .get_range(&hash, None, None, 10000, Default::default()) - .await?; - - for br in block_refs { - let version = match self - .garage - .version_table - .get(&br.version, &EmptyKey) - .await? - { - Some(v) => v, - None => continue, - }; - - if let Some(object) = self - .garage - .object_table - .get(&version.bucket_id, &version.key) - .await? - { - let ov = object.versions().iter().rev().find(|v| v.is_complete()); - if let Some(ov) = ov { - if ov.uuid == br.version { - let del_uuid = gen_uuid(); - let deleted_object = Object::new( - version.bucket_id, - version.key.clone(), - vec![ObjectVersion { - uuid: del_uuid, - timestamp: ov.timestamp + 1, - state: ObjectVersionState::Complete( - ObjectVersionData::DeleteMarker, - ), - }], - ); - self.garage.object_table.insert(&deleted_object).await?; - obj_dels += 1; - } - } - } - - if !version.deleted.get() { - let deleted_version = Version::new( - version.uuid, - version.bucket_id, - version.key.clone(), - true, - ); - self.garage.version_table.insert(&deleted_version).await?; - ver_dels += 1; - } - } - } - Ok(AdminRpc::Ok(format!( - "{} blocks were purged: {} object deletion markers added, {} versions marked deleted", - blocks.len(), - obj_dels, - ver_dels - ))) - } - } - } -} - -#[async_trait] -impl EndpointHandler for AdminRpcHandler { - async fn handle( - self: &Arc, - message: &AdminRpc, - _from: NodeID, - ) -> Result { - match message { - AdminRpc::BucketOperation(bo) => self.handle_bucket_cmd(bo).await, - AdminRpc::KeyOperation(ko) => self.handle_key_cmd(ko).await, - AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await, - AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await, - AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await, - AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await, - AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await, - m => Err(GarageError::unexpected_rpc_message(m).into()), - } - } -} diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs new file mode 100644 index 00000000..5f908ce4 --- /dev/null +++ b/src/garage/admin/block.rs @@ -0,0 +1,243 @@ +use garage_util::data::*; + +use garage_table::*; + +use garage_model::helper::error::{Error, OkOrBadRequest}; +use garage_model::s3::object_table::*; +use garage_model::s3::version_table::*; + +use crate::cli::*; + +use super::*; + +impl AdminRpcHandler { + pub(super) async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result { + match cmd { + BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList( + self.garage.block_manager.list_resync_errors()?, + )), + BlockOperation::Info { hash } => self.handle_block_info(hash).await, + BlockOperation::RetryNow { all, blocks } => { + self.handle_block_retry_now(*all, blocks).await + } + BlockOperation::Purge { yes, blocks } => self.handle_block_purge(*yes, blocks).await, + } + } + + async fn handle_block_info(&self, hash: &String) -> Result { + let hash = self.find_block_hash_by_prefix(hash)?; + let refcount = self.garage.block_manager.get_block_rc(&hash)?; + let block_refs = self + .garage + .block_ref_table + .get_range(&hash, None, None, 10000, Default::default()) + .await?; + let mut versions = vec![]; + let mut uploads = vec![]; + for br in block_refs { + if let Some(v) = self + .garage + .version_table + .get(&br.version, &EmptyKey) + .await? + { + if let VersionBacklink::MultipartUpload { upload_id } = &v.backlink { + if let Some(u) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? { + uploads.push(u); + } + } + versions.push(Ok(v)); + } else { + versions.push(Err(br.version)); + } + } + Ok(AdminRpc::BlockInfo { + hash, + refcount, + versions, + uploads, + }) + } + + async fn handle_block_retry_now( + &self, + all: bool, + blocks: &[String], + ) -> Result { + if all { + if !blocks.is_empty() { + return Err(Error::BadRequest( + "--all was specified, cannot also specify blocks".into(), + )); + } + let blocks = self.garage.block_manager.list_resync_errors()?; + for b in blocks.iter() { + self.garage.block_manager.resync.clear_backoff(&b.hash)?; + } + Ok(AdminRpc::Ok(format!( + "{} blocks returned in queue for a retry now (check logs to see results)", + blocks.len() + ))) + } else { + for hash in blocks { + let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; + let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; + self.garage.block_manager.resync.clear_backoff(&hash)?; + } + Ok(AdminRpc::Ok(format!( + "{} blocks returned in queue for a retry now (check logs to see results)", + blocks.len() + ))) + } + } + + async fn handle_block_purge(&self, yes: bool, blocks: &[String]) -> Result { + if !yes { + return Err(Error::BadRequest( + "Pass the --yes flag to confirm block purge operation.".into(), + )); + } + + let mut obj_dels = 0; + let mut mpu_dels = 0; + let mut ver_dels = 0; + let mut br_dels = 0; + + for hash in blocks { + let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; + let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; + let block_refs = self + .garage + .block_ref_table + .get_range(&hash, None, None, 10000, Default::default()) + .await?; + + for br in block_refs { + if let Some(version) = self + .garage + .version_table + .get(&br.version, &EmptyKey) + .await? + { + self.handle_block_purge_version_backlink( + &version, + &mut obj_dels, + &mut mpu_dels, + ) + .await?; + + if !version.deleted.get() { + let deleted_version = Version::new(version.uuid, version.backlink, true); + self.garage.version_table.insert(&deleted_version).await?; + ver_dels += 1; + } + } + if !br.deleted.get() { + let mut br = br; + br.deleted.set(); + self.garage.block_ref_table.insert(&br).await?; + br_dels += 1; + } + } + } + + Ok(AdminRpc::Ok(format!( + "Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted", + blocks.len(), + br_dels, + ver_dels, + obj_dels, + mpu_dels, + ))) + } + + async fn handle_block_purge_version_backlink( + &self, + version: &Version, + obj_dels: &mut usize, + mpu_dels: &mut usize, + ) -> Result<(), Error> { + let (bucket_id, key, ov_id) = match &version.backlink { + VersionBacklink::Object { bucket_id, key } => (*bucket_id, key.clone(), version.uuid), + VersionBacklink::MultipartUpload { upload_id } => { + if let Some(mut mpu) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? { + if !mpu.deleted.get() { + mpu.parts.clear(); + mpu.deleted.set(); + self.garage.mpu_table.insert(&mpu).await?; + *mpu_dels += 1; + } + (mpu.bucket_id, mpu.key.clone(), *upload_id) + } else { + return Ok(()); + } + } + }; + + if let Some(object) = self.garage.object_table.get(&bucket_id, &key).await? { + let ov = object.versions().iter().rev().find(|v| v.is_complete()); + if let Some(ov) = ov { + if ov.uuid == ov_id { + let del_uuid = gen_uuid(); + let deleted_object = Object::new( + bucket_id, + key, + vec![ObjectVersion { + uuid: del_uuid, + timestamp: ov.timestamp + 1, + state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker), + }], + ); + self.garage.object_table.insert(&deleted_object).await?; + *obj_dels += 1; + } + } + } + + Ok(()) + } + + // ---- helper function ---- + fn find_block_hash_by_prefix(&self, prefix: &str) -> Result { + if prefix.len() < 4 { + return Err(Error::BadRequest( + "Please specify at least 4 characters of the block hash".into(), + )); + } + + let prefix_bin = + hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?; + + let iter = self + .garage + .block_ref_table + .data + .store + .range(&prefix_bin[..]..) + .map_err(GarageError::from)?; + let mut found = None; + for item in iter { + let (k, _v) = item.map_err(GarageError::from)?; + let hash = Hash::try_from(&k[..32]).unwrap(); + if &hash.as_slice()[..prefix_bin.len()] != prefix_bin { + break; + } + if hex::encode(hash.as_slice()).starts_with(prefix) { + match &found { + Some(x) if *x == hash => (), + Some(_) => { + return Err(Error::BadRequest(format!( + "Several blocks match prefix `{}`", + prefix + ))); + } + None => { + found = Some(hash); + } + } + } + } + + found.ok_or_else(|| Error::BadRequest("No matching block found".into())) + } +} diff --git a/src/garage/admin/bucket.rs b/src/garage/admin/bucket.rs new file mode 100644 index 00000000..073329c1 --- /dev/null +++ b/src/garage/admin/bucket.rs @@ -0,0 +1,500 @@ +use std::collections::HashMap; +use std::fmt::Write; + +use garage_util::crdt::*; +use garage_util::time::*; + +use garage_table::*; + +use garage_model::bucket_alias_table::*; +use garage_model::bucket_table::*; +use garage_model::helper::error::{Error, OkOrBadRequest}; +use garage_model::permission::*; + +use crate::cli::*; + +use super::*; + +impl AdminRpcHandler { + pub(super) async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result { + match cmd { + BucketOperation::List => self.handle_list_buckets().await, + BucketOperation::Info(query) => self.handle_bucket_info(query).await, + BucketOperation::Create(query) => self.handle_create_bucket(&query.name).await, + BucketOperation::Delete(query) => self.handle_delete_bucket(query).await, + BucketOperation::Alias(query) => self.handle_alias_bucket(query).await, + BucketOperation::Unalias(query) => self.handle_unalias_bucket(query).await, + BucketOperation::Allow(query) => self.handle_bucket_allow(query).await, + BucketOperation::Deny(query) => self.handle_bucket_deny(query).await, + BucketOperation::Website(query) => self.handle_bucket_website(query).await, + BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await, + BucketOperation::CleanupIncompleteUploads(query) => { + self.handle_bucket_cleanup_incomplete_uploads(query).await + } + } + } + + async fn handle_list_buckets(&self) -> Result { + let buckets = self + .garage + .bucket_table + .get_range( + &EmptyKey, + None, + Some(DeletedFilter::NotDeleted), + 10000, + EnumerationOrder::Forward, + ) + .await?; + + Ok(AdminRpc::BucketList(buckets)) + } + + async fn handle_bucket_info(&self, query: &BucketOpt) -> Result { + let bucket_id = self + .garage + .bucket_helper() + .admin_get_existing_matching_bucket(&query.name) + .await?; + + let bucket = self + .garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + + let counters = self + .garage + .object_counter_table + .table + .get(&bucket_id, &EmptyKey) + .await? + .map(|x| x.filtered_values(&self.garage.system.cluster_layout())) + .unwrap_or_default(); + + let mpu_counters = self + .garage + .mpu_counter_table + .table + .get(&bucket_id, &EmptyKey) + .await? + .map(|x| x.filtered_values(&self.garage.system.cluster_layout())) + .unwrap_or_default(); + + let mut relevant_keys = HashMap::new(); + for (k, _) in bucket + .state + .as_option() + .unwrap() + .authorized_keys + .items() + .iter() + { + if let Some(key) = self + .garage + .key_table + .get(&EmptyKey, k) + .await? + .filter(|k| !k.is_deleted()) + { + relevant_keys.insert(k.clone(), key); + } + } + for ((k, _), _, _) in bucket + .state + .as_option() + .unwrap() + .local_aliases + .items() + .iter() + { + if relevant_keys.contains_key(k) { + continue; + } + if let Some(key) = self.garage.key_table.get(&EmptyKey, k).await? { + relevant_keys.insert(k.clone(), key); + } + } + + Ok(AdminRpc::BucketInfo { + bucket, + relevant_keys, + counters, + mpu_counters, + }) + } + + #[allow(clippy::ptr_arg)] + async fn handle_create_bucket(&self, name: &String) -> Result { + if !is_valid_bucket_name(name, self.garage.config.allow_punycode) { + return Err(Error::BadRequest(format!( + "{}: {}", + name, INVALID_BUCKET_NAME_MESSAGE + ))); + } + + let helper = self.garage.locked_helper().await; + + if let Some(alias) = self.garage.bucket_alias_table.get(&EmptyKey, name).await? { + if alias.state.get().is_some() { + return Err(Error::BadRequest(format!("Bucket {} already exists", name))); + } + } + + // ---- done checking, now commit ---- + + let bucket = Bucket::new(); + self.garage.bucket_table.insert(&bucket).await?; + + helper.set_global_bucket_alias(bucket.id, name).await?; + + Ok(AdminRpc::Ok(format!("Bucket {} was created.", name))) + } + + async fn handle_delete_bucket(&self, query: &DeleteBucketOpt) -> Result { + let helper = self.garage.locked_helper().await; + + let bucket_id = helper + .bucket() + .admin_get_existing_matching_bucket(&query.name) + .await?; + + // Get the alias, but keep in minde here the bucket name + // given in parameter can also be directly the bucket's ID. + // In that case bucket_alias will be None, and + // we can still delete the bucket if it has zero aliases + // (a condition which we try to prevent but that could still happen somehow). + // We just won't try to delete an alias entry because there isn't one. + let bucket_alias = self + .garage + .bucket_alias_table + .get(&EmptyKey, &query.name) + .await?; + + // Check bucket doesn't have other aliases + let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?; + let bucket_state = bucket.state.as_option().unwrap(); + if bucket_state + .aliases + .items() + .iter() + .filter(|(_, _, active)| *active) + .any(|(name, _, _)| name != &query.name) + { + return Err(Error::BadRequest(format!("Bucket {} still has other global aliases. Use `bucket unalias` to delete them one by one.", query.name))); + } + if bucket_state + .local_aliases + .items() + .iter() + .any(|(_, _, active)| *active) + { + return Err(Error::BadRequest(format!("Bucket {} still has other local aliases. Use `bucket unalias` to delete them one by one.", query.name))); + } + + // Check bucket is empty + if !helper.bucket().is_bucket_empty(bucket_id).await? { + return Err(Error::BadRequest(format!( + "Bucket {} is not empty", + query.name + ))); + } + + if !query.yes { + return Err(Error::BadRequest( + "Add --yes flag to really perform this operation".to_string(), + )); + } + + // --- done checking, now commit --- + // 1. delete authorization from keys that had access + for (key_id, _) in bucket.authorized_keys() { + helper + .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) + .await?; + } + + // 2. delete bucket alias + if bucket_alias.is_some() { + helper + .purge_global_bucket_alias(bucket_id, &query.name) + .await?; + } + + // 3. delete bucket + bucket.state = Deletable::delete(); + self.garage.bucket_table.insert(&bucket).await?; + + Ok(AdminRpc::Ok(format!("Bucket {} was deleted.", query.name))) + } + + async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result { + let helper = self.garage.locked_helper().await; + + let bucket_id = helper + .bucket() + .admin_get_existing_matching_bucket(&query.existing_bucket) + .await?; + + if let Some(key_pattern) = &query.local { + let key = helper.key().get_existing_matching_key(key_pattern).await?; + + helper + .set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name) + .await?; + Ok(AdminRpc::Ok(format!( + "Alias {} now points to bucket {:?} in namespace of key {}", + query.new_name, bucket_id, key.key_id + ))) + } else { + helper + .set_global_bucket_alias(bucket_id, &query.new_name) + .await?; + Ok(AdminRpc::Ok(format!( + "Alias {} now points to bucket {:?}", + query.new_name, bucket_id + ))) + } + } + + async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result { + let helper = self.garage.locked_helper().await; + + if let Some(key_pattern) = &query.local { + let key = helper.key().get_existing_matching_key(key_pattern).await?; + + let bucket_id = key + .state + .as_option() + .unwrap() + .local_aliases + .get(&query.name) + .cloned() + .flatten() + .ok_or_bad_request("Bucket not found")?; + + helper + .unset_local_bucket_alias(bucket_id, &key.key_id, &query.name) + .await?; + + Ok(AdminRpc::Ok(format!( + "Alias {} no longer points to bucket {:?} in namespace of key {}", + &query.name, bucket_id, key.key_id + ))) + } else { + let bucket_id = helper + .bucket() + .resolve_global_bucket_name(&query.name) + .await? + .ok_or_bad_request("Bucket not found")?; + + helper + .unset_global_bucket_alias(bucket_id, &query.name) + .await?; + + Ok(AdminRpc::Ok(format!( + "Alias {} no longer points to bucket {:?}", + &query.name, bucket_id + ))) + } + } + + async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result { + let helper = self.garage.locked_helper().await; + + let bucket_id = helper + .bucket() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + let key = helper + .key() + .get_existing_matching_key(&query.key_pattern) + .await?; + + let allow_read = query.read || key.allow_read(&bucket_id); + let allow_write = query.write || key.allow_write(&bucket_id); + let allow_owner = query.owner || key.allow_owner(&bucket_id); + + helper + .set_bucket_key_permissions( + bucket_id, + &key.key_id, + BucketKeyPerm { + timestamp: now_msec(), + allow_read, + allow_write, + allow_owner, + }, + ) + .await?; + + Ok(AdminRpc::Ok(format!( + "New permissions for {} on {}: read {}, write {}, owner {}.", + &key.key_id, &query.bucket, allow_read, allow_write, allow_owner + ))) + } + + async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result { + let helper = self.garage.locked_helper().await; + + let bucket_id = helper + .bucket() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + let key = helper + .key() + .get_existing_matching_key(&query.key_pattern) + .await?; + + let allow_read = !query.read && key.allow_read(&bucket_id); + let allow_write = !query.write && key.allow_write(&bucket_id); + let allow_owner = !query.owner && key.allow_owner(&bucket_id); + + helper + .set_bucket_key_permissions( + bucket_id, + &key.key_id, + BucketKeyPerm { + timestamp: now_msec(), + allow_read, + allow_write, + allow_owner, + }, + ) + .await?; + + Ok(AdminRpc::Ok(format!( + "New permissions for {} on {}: read {}, write {}, owner {}.", + &key.key_id, &query.bucket, allow_read, allow_write, allow_owner + ))) + } + + async fn handle_bucket_website(&self, query: &WebsiteOpt) -> Result { + let bucket_id = self + .garage + .bucket_helper() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + + let mut bucket = self + .garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + let bucket_state = bucket.state.as_option_mut().unwrap(); + + if !(query.allow ^ query.deny) { + return Err(Error::BadRequest( + "You must specify exactly one flag, either --allow or --deny".to_string(), + )); + } + + let website = if query.allow { + Some(WebsiteConfig { + index_document: query.index_document.clone(), + error_document: query.error_document.clone(), + }) + } else { + None + }; + + bucket_state.website_config.update(website); + self.garage.bucket_table.insert(&bucket).await?; + + let msg = if query.allow { + format!("Website access allowed for {}", &query.bucket) + } else { + format!("Website access denied for {}", &query.bucket) + }; + + Ok(AdminRpc::Ok(msg)) + } + + async fn handle_bucket_set_quotas(&self, query: &SetQuotasOpt) -> Result { + let bucket_id = self + .garage + .bucket_helper() + .admin_get_existing_matching_bucket(&query.bucket) + .await?; + + let mut bucket = self + .garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + let bucket_state = bucket.state.as_option_mut().unwrap(); + + if query.max_size.is_none() && query.max_objects.is_none() { + return Err(Error::BadRequest( + "You must specify either --max-size or --max-objects (or both) for this command to do something.".to_string(), + )); + } + + let mut quotas = bucket_state.quotas.get().clone(); + + match query.max_size.as_ref().map(String::as_ref) { + Some("none") => quotas.max_size = None, + Some(v) => { + let bs = v + .parse::() + .ok_or_bad_request(format!("Invalid size specified: {}", v))?; + quotas.max_size = Some(bs.as_u64()); + } + _ => (), + } + + match query.max_objects.as_ref().map(String::as_ref) { + Some("none") => quotas.max_objects = None, + Some(v) => { + let mo = v + .parse::() + .ok_or_bad_request(format!("Invalid number specified: {}", v))?; + quotas.max_objects = Some(mo); + } + _ => (), + } + + bucket_state.quotas.update(quotas); + self.garage.bucket_table.insert(&bucket).await?; + + Ok(AdminRpc::Ok(format!( + "Quotas updated for {}", + &query.bucket + ))) + } + + async fn handle_bucket_cleanup_incomplete_uploads( + &self, + query: &CleanupIncompleteUploadsOpt, + ) -> Result { + let mut bucket_ids = vec![]; + for b in query.buckets.iter() { + bucket_ids.push( + self.garage + .bucket_helper() + .admin_get_existing_matching_bucket(b) + .await?, + ); + } + + let duration = parse_duration::parse::parse(&query.older_than) + .ok_or_bad_request("Invalid duration passed for --older-than parameter")?; + + let mut ret = String::new(); + for bucket in bucket_ids { + let count = self + .garage + .bucket_helper() + .cleanup_incomplete_uploads(&bucket, duration) + .await?; + writeln!( + &mut ret, + "Bucket {:?}: {} incomplete uploads aborted", + bucket, count + ) + .unwrap(); + } + + Ok(AdminRpc::Ok(ret)) + } +} diff --git a/src/garage/admin/key.rs b/src/garage/admin/key.rs new file mode 100644 index 00000000..bd010d2c --- /dev/null +++ b/src/garage/admin/key.rs @@ -0,0 +1,161 @@ +use std::collections::HashMap; + +use garage_table::*; + +use garage_model::helper::error::*; +use garage_model::key_table::*; + +use crate::cli::*; + +use super::*; + +impl AdminRpcHandler { + pub(super) async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result { + match cmd { + KeyOperation::List => self.handle_list_keys().await, + KeyOperation::Info(query) => self.handle_key_info(query).await, + KeyOperation::Create(query) => self.handle_create_key(query).await, + KeyOperation::Rename(query) => self.handle_rename_key(query).await, + KeyOperation::Delete(query) => self.handle_delete_key(query).await, + KeyOperation::Allow(query) => self.handle_allow_key(query).await, + KeyOperation::Deny(query) => self.handle_deny_key(query).await, + KeyOperation::Import(query) => self.handle_import_key(query).await, + } + } + + async fn handle_list_keys(&self) -> Result { + let key_ids = self + .garage + .key_table + .get_range( + &EmptyKey, + None, + Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)), + 10000, + EnumerationOrder::Forward, + ) + .await? + .iter() + .map(|k| (k.key_id.to_string(), k.params().unwrap().name.get().clone())) + .collect::>(); + Ok(AdminRpc::KeyList(key_ids)) + } + + async fn handle_key_info(&self, query: &KeyInfoOpt) -> Result { + let mut key = self + .garage + .key_helper() + .get_existing_matching_key(&query.key_pattern) + .await?; + + if !query.show_secret { + key.state.as_option_mut().unwrap().secret_key = "(redacted)".into(); + } + + self.key_info_result(key).await + } + + async fn handle_create_key(&self, query: &KeyNewOpt) -> Result { + let key = Key::new(&query.name); + self.garage.key_table.insert(&key).await?; + self.key_info_result(key).await + } + + async fn handle_rename_key(&self, query: &KeyRenameOpt) -> Result { + let mut key = self + .garage + .key_helper() + .get_existing_matching_key(&query.key_pattern) + .await?; + key.params_mut() + .unwrap() + .name + .update(query.new_name.clone()); + self.garage.key_table.insert(&key).await?; + self.key_info_result(key).await + } + + async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result { + let helper = self.garage.locked_helper().await; + + let mut key = helper + .key() + .get_existing_matching_key(&query.key_pattern) + .await?; + + if !query.yes { + return Err(Error::BadRequest( + "Add --yes flag to really perform this operation".to_string(), + )); + } + + helper.delete_key(&mut key).await?; + + Ok(AdminRpc::Ok(format!( + "Key {} was deleted successfully.", + key.key_id + ))) + } + + async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result { + let mut key = self + .garage + .key_helper() + .get_existing_matching_key(&query.key_pattern) + .await?; + if query.create_bucket { + key.params_mut().unwrap().allow_create_bucket.update(true); + } + self.garage.key_table.insert(&key).await?; + self.key_info_result(key).await + } + + async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result { + let mut key = self + .garage + .key_helper() + .get_existing_matching_key(&query.key_pattern) + .await?; + if query.create_bucket { + key.params_mut().unwrap().allow_create_bucket.update(false); + } + self.garage.key_table.insert(&key).await?; + self.key_info_result(key).await + } + + async fn handle_import_key(&self, query: &KeyImportOpt) -> Result { + if !query.yes { + return Err(Error::BadRequest("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key new` instead. Add the --yes flag if you really want to re-import a key.".to_string())); + } + + let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?; + if prev_key.is_some() { + return Err(Error::BadRequest(format!("Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", query.key_id))); + } + + let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name) + .ok_or_bad_request("Invalid key format")?; + self.garage.key_table.insert(&imported_key).await?; + + self.key_info_result(imported_key).await + } + + async fn key_info_result(&self, key: Key) -> Result { + let mut relevant_buckets = HashMap::new(); + + for (id, _) in key + .state + .as_option() + .unwrap() + .authorized_buckets + .items() + .iter() + { + if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? { + relevant_buckets.insert(*id, b); + } + } + + Ok(AdminRpc::KeyInfo(key, relevant_buckets)) + } +} diff --git a/src/garage/admin/mod.rs b/src/garage/admin/mod.rs new file mode 100644 index 00000000..6ae8fa88 --- /dev/null +++ b/src/garage/admin/mod.rs @@ -0,0 +1,545 @@ +mod block; +mod bucket; +mod key; + +use std::collections::HashMap; +use std::fmt::Write; +use std::future::Future; +use std::sync::Arc; + +use futures::future::FutureExt; + +use serde::{Deserialize, Serialize}; + +use format_table::format_table_to_string; + +use garage_util::background::BackgroundRunner; +use garage_util::data::*; +use garage_util::error::Error as GarageError; + +use garage_table::replication::*; +use garage_table::*; + +use garage_rpc::layout::PARTITION_BITS; +use garage_rpc::*; + +use garage_block::manager::BlockResyncErrorInfo; + +use garage_model::bucket_table::*; +use garage_model::garage::Garage; +use garage_model::helper::error::{Error, OkOrBadRequest}; +use garage_model::key_table::*; +use garage_model::s3::mpu_table::MultipartUpload; +use garage_model::s3::version_table::Version; + +use crate::cli::*; +use crate::repair::online::launch_online_repair; + +pub const ADMIN_RPC_PATH: &str = "garage/admin_rpc.rs/Rpc"; + +#[derive(Debug, Serialize, Deserialize)] +#[allow(clippy::large_enum_variant)] +pub enum AdminRpc { + BucketOperation(BucketOperation), + KeyOperation(KeyOperation), + LaunchRepair(RepairOpt), + Stats(StatsOpt), + Worker(WorkerOperation), + BlockOperation(BlockOperation), + MetaOperation(MetaOperation), + + // Replies + Ok(String), + BucketList(Vec), + BucketInfo { + bucket: Bucket, + relevant_keys: HashMap, + counters: HashMap, + mpu_counters: HashMap, + }, + KeyList(Vec<(String, String)>), + KeyInfo(Key, HashMap), + WorkerList( + HashMap, + WorkerListOpt, + ), + WorkerVars(Vec<(Uuid, String, String)>), + WorkerInfo(usize, garage_util::background::WorkerInfo), + BlockErrorList(Vec), + BlockInfo { + hash: Hash, + refcount: u64, + versions: Vec>, + uploads: Vec, + }, +} + +impl Rpc for AdminRpc { + type Response = Result; +} + +pub struct AdminRpcHandler { + garage: Arc, + background: Arc, + endpoint: Arc>, +} + +impl AdminRpcHandler { + pub fn new(garage: Arc, background: Arc) -> Arc { + let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into()); + let admin = Arc::new(Self { + garage, + background, + endpoint, + }); + admin.endpoint.set_handler(admin.clone()); + admin + } + + // ================ REPAIR COMMANDS ==================== + + async fn handle_launch_repair(self: &Arc, opt: RepairOpt) -> Result { + if !opt.yes { + return Err(Error::BadRequest( + "Please provide the --yes flag to initiate repair operations.".to_string(), + )); + } + if opt.all_nodes { + let mut opt_to_send = opt.clone(); + opt_to_send.all_nodes = false; + + let mut failures = vec![]; + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { + let node = (*node).into(); + let resp = self + .endpoint + .call( + &node, + AdminRpc::LaunchRepair(opt_to_send.clone()), + PRIO_NORMAL, + ) + .await; + if !matches!(resp, Ok(Ok(_))) { + failures.push(node); + } + } + if failures.is_empty() { + Ok(AdminRpc::Ok("Repair launched on all nodes".to_string())) + } else { + Err(Error::BadRequest(format!( + "Could not launch repair on nodes: {:?} (launched successfully on other nodes)", + failures + ))) + } + } else { + launch_online_repair(&self.garage, &self.background, opt).await?; + Ok(AdminRpc::Ok(format!( + "Repair launched on {:?}", + self.garage.system.id + ))) + } + } + + // ================ STATS COMMANDS ==================== + + async fn handle_stats(&self, opt: StatsOpt) -> Result { + if opt.all_nodes { + let mut ret = String::new(); + let mut all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in self.garage.system.get_known_nodes().iter() { + if node.is_up && !all_nodes.contains(&node.id) { + all_nodes.push(node.id); + } + } + + for node in all_nodes.iter() { + let mut opt = opt.clone(); + opt.all_nodes = false; + opt.skip_global = true; + + writeln!(&mut ret, "\n======================").unwrap(); + writeln!(&mut ret, "Stats for node {:?}:", node).unwrap(); + + let node_id = (*node).into(); + match self + .endpoint + .call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL) + .await + { + Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(), + Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(), + Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(), + Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(), + } + } + + writeln!(&mut ret, "\n======================").unwrap(); + write!( + &mut ret, + "Cluster statistics:\n\n{}", + self.gather_cluster_stats() + ) + .unwrap(); + + Ok(AdminRpc::Ok(ret)) + } else { + Ok(AdminRpc::Ok(self.gather_stats_local(opt)?)) + } + } + + fn gather_stats_local(&self, opt: StatsOpt) -> Result { + let mut ret = String::new(); + writeln!( + &mut ret, + "\nGarage version: {} [features: {}]\nRust compiler version: {}", + garage_util::version::garage_version(), + garage_util::version::garage_features() + .map(|list| list.join(", ")) + .unwrap_or_else(|| "(unknown)".into()), + garage_util::version::rust_version(), + ) + .unwrap(); + + writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap(); + + // Gather table statistics + let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tGcTodo".into()]; + table.push(self.gather_table_stats(&self.garage.bucket_table)?); + table.push(self.gather_table_stats(&self.garage.key_table)?); + table.push(self.gather_table_stats(&self.garage.object_table)?); + table.push(self.gather_table_stats(&self.garage.version_table)?); + table.push(self.gather_table_stats(&self.garage.block_ref_table)?); + write!( + &mut ret, + "\nTable stats:\n{}", + format_table_to_string(table) + ) + .unwrap(); + + // Gather block manager statistics + writeln!(&mut ret, "\nBlock manager stats:").unwrap(); + let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string(); + + writeln!( + &mut ret, + " number of RC entries (~= number of blocks): {}", + rc_len + ) + .unwrap(); + writeln!( + &mut ret, + " resync queue length: {}", + self.garage.block_manager.resync.queue_approximate_len()? + ) + .unwrap(); + writeln!( + &mut ret, + " blocks with resync errors: {}", + self.garage.block_manager.resync.errors_approximate_len()? + ) + .unwrap(); + + if !opt.skip_global { + write!(&mut ret, "\n{}", self.gather_cluster_stats()).unwrap(); + } + + Ok(ret) + } + + fn gather_cluster_stats(&self) -> String { + let mut ret = String::new(); + + // Gather storage node and free space statistics for current nodes + let layout = &self.garage.system.cluster_layout(); + let mut node_partition_count = HashMap::::new(); + for short_id in layout.current().ring_assignment_data.iter() { + let id = layout.current().node_id_vec[*short_id as usize]; + *node_partition_count.entry(id).or_default() += 1; + } + let node_info = self + .garage + .system + .get_known_nodes() + .into_iter() + .map(|n| (n.id, n)) + .collect::>(); + + let mut table = vec![" ID\tHostname\tZone\tCapacity\tPart.\tDataAvail\tMetaAvail".into()]; + for (id, parts) in node_partition_count.iter() { + let info = node_info.get(id); + let status = info.map(|x| &x.status); + let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref()); + let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?"); + let zone = role.map(|x| x.zone.as_str()).unwrap_or("?"); + let capacity = role + .map(|x| x.capacity_string()) + .unwrap_or_else(|| "?".into()); + let avail_str = |x| match x { + Some((avail, total)) => { + let pct = (avail as f64) / (total as f64) * 100.; + let avail = bytesize::ByteSize::b(avail); + let total = bytesize::ByteSize::b(total); + format!("{}/{} ({:.1}%)", avail, total, pct) + } + None => "?".into(), + }; + let data_avail = avail_str(status.and_then(|x| x.data_disk_avail)); + let meta_avail = avail_str(status.and_then(|x| x.meta_disk_avail)); + table.push(format!( + " {:?}\t{}\t{}\t{}\t{}\t{}\t{}", + id, hostname, zone, capacity, parts, data_avail, meta_avail + )); + } + write!( + &mut ret, + "Storage nodes:\n{}", + format_table_to_string(table) + ) + .unwrap(); + + let meta_part_avail = node_partition_count + .iter() + .filter_map(|(id, parts)| { + node_info + .get(id) + .and_then(|x| x.status.meta_disk_avail) + .map(|c| c.0 / *parts) + }) + .collect::>(); + let data_part_avail = node_partition_count + .iter() + .filter_map(|(id, parts)| { + node_info + .get(id) + .and_then(|x| x.status.data_disk_avail) + .map(|c| c.0 / *parts) + }) + .collect::>(); + if !meta_part_avail.is_empty() && !data_part_avail.is_empty() { + let meta_avail = + bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); + let data_avail = + bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); + writeln!( + &mut ret, + "\nEstimated available storage space cluster-wide (might be lower in practice):" + ) + .unwrap(); + if meta_part_avail.len() < node_partition_count.len() + || data_part_avail.len() < node_partition_count.len() + { + writeln!(&mut ret, " data: < {}", data_avail).unwrap(); + writeln!(&mut ret, " metadata: < {}", meta_avail).unwrap(); + writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap(); + } else { + writeln!(&mut ret, " data: {}", data_avail).unwrap(); + writeln!(&mut ret, " metadata: {}", meta_avail).unwrap(); + } + } + + ret + } + + fn gather_table_stats(&self, t: &Arc>) -> Result + where + F: TableSchema + 'static, + R: TableReplication + 'static, + { + let data_len = t + .data + .store + .approximate_len() + .map_err(GarageError::from)? + .to_string(); + let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string(); + + Ok(format!( + " {}\t{}\t{}\t{}\t{}", + F::TABLE_NAME, + data_len, + mkl_len, + t.merkle_updater.todo_approximate_len()?, + t.data.gc_todo_approximate_len()? + )) + } + + // ================ WORKER COMMANDS ==================== + + async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result { + match cmd { + WorkerOperation::List { opt } => { + let workers = self.background.get_worker_info(); + Ok(AdminRpc::WorkerList(workers, *opt)) + } + WorkerOperation::Info { tid } => { + let info = self + .background + .get_worker_info() + .get(tid) + .ok_or_bad_request(format!("No worker with TID {}", tid))? + .clone(); + Ok(AdminRpc::WorkerInfo(*tid, info)) + } + WorkerOperation::Get { + all_nodes, + variable, + } => self.handle_get_var(*all_nodes, variable).await, + WorkerOperation::Set { + all_nodes, + variable, + value, + } => self.handle_set_var(*all_nodes, variable, value).await, + } + } + + async fn handle_get_var( + &self, + all_nodes: bool, + variable: &Option, + ) -> Result { + if all_nodes { + let mut ret = vec![]; + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { + let node = (*node).into(); + match self + .endpoint + .call( + &node, + AdminRpc::Worker(WorkerOperation::Get { + all_nodes: false, + variable: variable.clone(), + }), + PRIO_NORMAL, + ) + .await?? + { + AdminRpc::WorkerVars(v) => ret.extend(v), + m => return Err(GarageError::unexpected_rpc_message(m).into()), + } + } + Ok(AdminRpc::WorkerVars(ret)) + } else { + #[allow(clippy::collapsible_else_if)] + if let Some(v) = variable { + Ok(AdminRpc::WorkerVars(vec![( + self.garage.system.id, + v.clone(), + self.garage.bg_vars.get(v)?, + )])) + } else { + let mut vars = self.garage.bg_vars.get_all(); + vars.sort(); + Ok(AdminRpc::WorkerVars( + vars.into_iter() + .map(|(k, v)| (self.garage.system.id, k.to_string(), v)) + .collect(), + )) + } + } + } + + async fn handle_set_var( + &self, + all_nodes: bool, + variable: &str, + value: &str, + ) -> Result { + if all_nodes { + let mut ret = vec![]; + let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec(); + for node in all_nodes.iter() { + let node = (*node).into(); + match self + .endpoint + .call( + &node, + AdminRpc::Worker(WorkerOperation::Set { + all_nodes: false, + variable: variable.to_string(), + value: value.to_string(), + }), + PRIO_NORMAL, + ) + .await?? + { + AdminRpc::WorkerVars(v) => ret.extend(v), + m => return Err(GarageError::unexpected_rpc_message(m).into()), + } + } + Ok(AdminRpc::WorkerVars(ret)) + } else { + self.garage.bg_vars.set(variable, value)?; + Ok(AdminRpc::WorkerVars(vec![( + self.garage.system.id, + variable.to_string(), + value.to_string(), + )])) + } + } + + // ================ META DB COMMANDS ==================== + + async fn handle_meta_cmd(self: &Arc, mo: &MetaOperation) -> Result { + match mo { + MetaOperation::Snapshot { all: true } => { + let to = self.garage.system.cluster_layout().all_nodes().to_vec(); + + let resps = futures::future::join_all(to.iter().map(|to| async move { + let to = (*to).into(); + self.endpoint + .call( + &to, + AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }), + PRIO_NORMAL, + ) + .await? + })) + .await; + + let mut ret = vec![]; + for (to, resp) in to.iter().zip(resps.iter()) { + let res_str = match resp { + Ok(_) => "ok".to_string(), + Err(e) => format!("error: {}", e), + }; + ret.push(format!("{:?}\t{}", to, res_str)); + } + + if resps.iter().any(Result::is_err) { + Err(GarageError::Message(format_table_to_string(ret)).into()) + } else { + Ok(AdminRpc::Ok(format_table_to_string(ret))) + } + } + MetaOperation::Snapshot { all: false } => { + garage_model::snapshot::async_snapshot_metadata(&self.garage).await?; + Ok(AdminRpc::Ok("Snapshot has been saved.".into())) + } + } + } +} + +impl EndpointHandler for AdminRpcHandler { + fn handle( + self: &Arc, + message: &AdminRpc, + _from: NodeID, + ) -> impl Future> + Send { + let self2 = self.clone(); + async move { + match message { + AdminRpc::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await, + AdminRpc::KeyOperation(ko) => self2.handle_key_cmd(ko).await, + AdminRpc::LaunchRepair(opt) => self2.handle_launch_repair(opt.clone()).await, + AdminRpc::Stats(opt) => self2.handle_stats(opt.clone()).await, + AdminRpc::Worker(wo) => self2.handle_worker_cmd(wo).await, + AdminRpc::BlockOperation(bo) => self2.handle_block_cmd(bo).await, + AdminRpc::MetaOperation(mo) => self2.handle_meta_cmd(mo).await, + m => Err(GarageError::unexpected_rpc_message(m).into()), + } + } + .boxed() + } +} diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index 0d735885..44d3d96c 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -1,8 +1,8 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::time::Duration; +use format_table::format_table; use garage_util::error::*; -use garage_util::formater::format_table; use garage_rpc::layout::*; use garage_rpc::system::*; @@ -33,9 +33,6 @@ pub async fn cli_command_dispatch( Command::Key(ko) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::KeyOperation(ko)).await } - Command::Migrate(mo) => { - cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Migrate(mo)).await - } Command::Repair(ro) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::LaunchRepair(ro)).await } @@ -44,56 +41,74 @@ pub async fn cli_command_dispatch( Command::Block(bo) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await } + Command::Meta(mo) => { + cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::MetaOperation(mo)).await + } _ => unreachable!(), } } pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> Result<(), Error> { - let status = match rpc_cli - .call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL) - .await?? - { - SystemRpc::ReturnKnownNodes(nodes) => nodes, - resp => return Err(Error::Message(format!("Invalid RPC response: {:?}", resp))), - }; + let status = fetch_status(rpc_cli, rpc_host).await?; let layout = fetch_layout(rpc_cli, rpc_host).await?; println!("==== HEALTHY NODES ===="); let mut healthy_nodes = vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tDataAvail".to_string()]; for adv in status.iter().filter(|adv| adv.is_up) { - match layout.roles.get(&adv.id) { - Some(NodeRoleV(Some(cfg))) => { - let data_avail = match &adv.status.data_disk_avail { - _ if cfg.capacity.is_none() => "N/A".into(), - Some((avail, total)) => { - let pct = (*avail as f64) / (*total as f64) * 100.; - let avail = bytesize::ByteSize::b(*avail); - format!("{} ({:.1}%)", avail, pct) - } - None => "?".into(), - }; + let host = adv.status.hostname.as_deref().unwrap_or("?"); + let addr = match adv.addr { + Some(addr) => addr.to_string(), + None => "N/A".to_string(), + }; + if let Some(NodeRoleV(Some(cfg))) = layout.current().roles.get(&adv.id) { + let data_avail = match &adv.status.data_disk_avail { + _ if cfg.capacity.is_none() => "N/A".into(), + Some((avail, total)) => { + let pct = (*avail as f64) / (*total as f64) * 100.; + let avail = bytesize::ByteSize::b(*avail); + format!("{} ({:.1}%)", avail, pct) + } + None => "?".into(), + }; + healthy_nodes.push(format!( + "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}", + id = adv.id, + host = host, + addr = addr, + tags = cfg.tags.join(","), + zone = cfg.zone, + capacity = cfg.capacity_string(), + data_avail = data_avail, + )); + } else { + let prev_role = layout + .versions + .iter() + .rev() + .find_map(|x| match x.roles.get(&adv.id) { + Some(NodeRoleV(Some(cfg))) => Some(cfg), + _ => None, + }); + if let Some(cfg) = prev_role { healthy_nodes.push(format!( - "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}", + "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\tdraining metadata...", id = adv.id, - host = adv.status.hostname, - addr = adv.addr, + host = host, + addr = addr, tags = cfg.tags.join(","), zone = cfg.zone, - capacity = cfg.capacity_string(), - data_avail = data_avail, )); - } - _ => { - let new_role = match layout.staging.get(&adv.id) { - Some(NodeRoleV(Some(_))) => "(pending)", + } else { + let new_role = match layout.staging.get().roles.get(&adv.id) { + Some(NodeRoleV(Some(_))) => "pending...", _ => "NO ROLE ASSIGNED", }; healthy_nodes.push(format!( - "{id:?}\t{h}\t{addr}\t{new_role}", + "{id:?}\t{h}\t{addr}\t\t\t{new_role}", id = adv.id, - h = adv.status.hostname, - addr = adv.addr, + h = host, + addr = addr, new_role = new_role, )); } @@ -101,51 +116,73 @@ pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> } format_table(healthy_nodes); - let status_keys = status.iter().map(|adv| adv.id).collect::>(); - let failure_case_1 = status + // Determine which nodes are unhealthy and print that to stdout + let status_map = status .iter() - .any(|adv| !adv.is_up && matches!(layout.roles.get(&adv.id), Some(NodeRoleV(Some(_))))); - let failure_case_2 = layout - .roles - .items() - .iter() - .any(|(id, _, v)| !status_keys.contains(id) && v.0.is_some()); - if failure_case_1 || failure_case_2 { - println!("\n==== FAILED NODES ===="); - let mut failed_nodes = - vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tLast seen".to_string()]; - for adv in status.iter().filter(|adv| !adv.is_up) { - if let Some(NodeRoleV(Some(cfg))) = layout.roles.get(&adv.id) { - let tf = timeago::Formatter::new(); - failed_nodes.push(format!( - "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}", - id = adv.id, - host = adv.status.hostname, - addr = adv.addr, - tags = cfg.tags.join(","), - zone = cfg.zone, - capacity = cfg.capacity_string(), - last_seen = adv - .last_seen_secs_ago + .map(|adv| (adv.id, adv)) + .collect::>(); + + let tf = timeago::Formatter::new(); + let mut drain_msg = false; + let mut failed_nodes = vec!["ID\tHostname\tTags\tZone\tCapacity\tLast seen".to_string()]; + let mut listed = HashSet::new(); + for ver in layout.versions.iter().rev() { + for (node, _, role) in ver.roles.items().iter() { + let cfg = match role { + NodeRoleV(Some(role)) if role.capacity.is_some() => role, + _ => continue, + }; + + if listed.contains(node) { + continue; + } + listed.insert(*node); + + let adv = status_map.get(node); + if adv.map(|x| x.is_up).unwrap_or(false) { + continue; + } + + // Node is in a layout version, is not a gateway node, and is not up: + // it is in a failed state, add proper line to the output + let (host, last_seen) = match adv { + Some(adv) => ( + adv.status.hostname.as_deref().unwrap_or("?"), + adv.last_seen_secs_ago .map(|s| tf.convert(Duration::from_secs(s))) .unwrap_or_else(|| "never seen".into()), - )); - } - } - for (id, _, role_v) in layout.roles.items().iter() { - if let NodeRoleV(Some(cfg)) = role_v { - if !status_keys.contains(id) { - failed_nodes.push(format!( - "{id:?}\t??\t??\t[{tags}]\t{zone}\t{capacity}\tnever seen", - id = id, - tags = cfg.tags.join(","), - zone = cfg.zone, - capacity = cfg.capacity_string(), - )); - } - } + ), + None => ("??", "never seen".into()), + }; + let capacity = if ver.version == layout.current().version { + cfg.capacity_string() + } else { + drain_msg = true; + "draining metadata...".to_string() + }; + failed_nodes.push(format!( + "{id:?}\t{host}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}", + id = node, + host = host, + tags = cfg.tags.join(","), + zone = cfg.zone, + capacity = capacity, + last_seen = last_seen, + )); } + } + + if failed_nodes.len() > 1 { + println!("\n==== FAILED NODES ===="); format_table(failed_nodes); + if drain_msg { + println!(); + println!("Your cluster is expecting to drain data from nodes that are currently unavailable."); + println!("If these nodes are definitely dead, please review the layout history with"); + println!( + "`garage layout history` and use `garage layout skip-dead-nodes` to force progress." + ); + } } if print_staging_role_changes(&layout) { @@ -190,8 +227,9 @@ pub async fn cmd_admin( bucket, relevant_keys, counters, + mpu_counters, } => { - print_bucket_info(&bucket, &relevant_keys, &counters); + print_bucket_info(&bucket, &relevant_keys, &counters, &mpu_counters); } AdminRpc::KeyList(kl) => { print_key_list(kl); @@ -215,8 +253,9 @@ pub async fn cmd_admin( hash, refcount, versions, + uploads, } => { - print_block_info(hash, refcount, versions); + print_block_info(hash, refcount, versions, uploads); } r => { error!("Unexpected response: {:?}", r); @@ -224,3 +263,18 @@ pub async fn cmd_admin( } Ok(()) } + +// ---- utility ---- + +pub async fn fetch_status( + rpc_cli: &Endpoint, + rpc_host: NodeID, +) -> Result, Error> { + match rpc_cli + .call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL) + .await?? + { + SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes), + resp => Err(Error::unexpected_rpc_message(resp)), + } +} diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs new file mode 100644 index 00000000..a40fb61f --- /dev/null +++ b/src/garage/cli/convert_db.rs @@ -0,0 +1,65 @@ +use std::path::PathBuf; + +use structopt::StructOpt; + +use garage_db::*; + +/// K2V command line interface +#[derive(StructOpt, Debug)] +pub struct ConvertDbOpt { + /// Input database path (not the same as metadata_dir, see + /// https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0) + #[structopt(short = "i")] + input_path: PathBuf, + /// Input database engine (lmdb or sqlite; limited by db engines + /// enabled in this build) + #[structopt(short = "a")] + input_engine: Engine, + + /// Output database path + #[structopt(short = "o")] + output_path: PathBuf, + /// Output database engine + #[structopt(short = "b")] + output_engine: Engine, + + #[structopt(flatten)] + #[allow(dead_code)] + db_open: OpenDbOpt, +} + +/// Overrides for database open operation +#[derive(StructOpt, Debug, Default)] +pub struct OpenDbOpt { + #[cfg(feature = "lmdb")] + #[structopt(flatten)] + lmdb: OpenLmdbOpt, +} + +/// Overrides for LMDB database open operation +#[cfg(feature = "lmdb")] +#[derive(StructOpt, Debug, Default)] +pub struct OpenLmdbOpt { + /// LMDB map size override + /// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB) + #[cfg(feature = "lmdb")] + #[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)] + map_size: Option, +} + +pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { + if args.input_engine == args.output_engine { + return Err(Error("input and output database engine must differ".into())); + } + + let opt = OpenOpt { + #[cfg(feature = "lmdb")] + lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize), + ..Default::default() + }; + + let input = open_db(&args.input_path, args.input_engine, &opt)?; + let output = open_db(&args.output_path, args.output_engine, &opt)?; + output.import(&input)?; + Ok(()) +} diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs index 20813f1c..43ca5c09 100644 --- a/src/garage/cli/init.rs +++ b/src/garage/cli/init.rs @@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { idstr ); eprintln!( - "where is their own node identifier in the format: @:" + "where is their own node identifier in the format: @:" ); eprintln!(); eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:"); diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs index 3884bb92..f053eef4 100644 --- a/src/garage/cli/layout.rs +++ b/src/garage/cli/layout.rs @@ -1,6 +1,8 @@ +use bytesize::ByteSize; + +use format_table::format_table; use garage_util::crdt::Crdt; use garage_util::error::*; -use garage_util::formater::format_table; use garage_rpc::layout::*; use garage_rpc::system::*; @@ -14,8 +16,8 @@ pub async fn cli_layout_command_dispatch( rpc_host: NodeID, ) -> Result<(), Error> { match cmd { - LayoutOperation::Assign(configure_opt) => { - cmd_assign_role(system_rpc_endpoint, rpc_host, configure_opt).await + LayoutOperation::Assign(assign_opt) => { + cmd_assign_role(system_rpc_endpoint, rpc_host, assign_opt).await } LayoutOperation::Remove(remove_opt) => { cmd_remove_role(system_rpc_endpoint, rpc_host, remove_opt).await @@ -27,6 +29,13 @@ pub async fn cli_layout_command_dispatch( LayoutOperation::Revert(revert_opt) => { cmd_revert_layout(system_rpc_endpoint, rpc_host, revert_opt).await } + LayoutOperation::Config(config_opt) => { + cmd_config_layout(system_rpc_endpoint, rpc_host, config_opt).await + } + LayoutOperation::History => cmd_layout_history(system_rpc_endpoint, rpc_host).await, + LayoutOperation::SkipDeadNodes(assume_sync_opt) => { + cmd_layout_skip_dead_nodes(system_rpc_endpoint, rpc_host, assume_sync_opt).await + } } } @@ -44,6 +53,7 @@ pub async fn cmd_assign_role( }; let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + let all_nodes = layout.get_all_nodes(); let added_nodes = args .node_ids @@ -53,21 +63,23 @@ pub async fn cmd_assign_role( status .iter() .map(|adv| adv.id) - .chain(layout.node_ids().iter().cloned()), + .chain(all_nodes.iter().cloned()), node_id, ) }) .collect::, _>>()?; - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); for replaced in args.replace.iter() { - let replaced_node = find_matching_node(layout.node_ids().iter().cloned(), replaced)?; + let replaced_node = find_matching_node(all_nodes.iter().cloned(), replaced)?; match roles.get(&replaced_node) { Some(NodeRoleV(Some(_))) => { layout .staging + .get_mut() + .roles .merge(&roles.update_mutator(replaced_node, NodeRoleV(None))); } _ => { @@ -83,7 +95,7 @@ pub async fn cmd_assign_role( return Err(Error::Message( "-c and -g are mutually exclusive, please configure node either with c>0 to act as a storage node or with -g to act as a gateway node".into())); } - if args.capacity == Some(0) { + if args.capacity == Some(ByteSize::b(0)) { return Err(Error::Message("Invalid capacity value: 0".into())); } @@ -91,7 +103,7 @@ pub async fn cmd_assign_role( let new_entry = match roles.get(&added_node) { Some(NodeRoleV(Some(old))) => { let capacity = match args.capacity { - Some(c) => Some(c), + Some(c) => Some(c.as_u64()), None if args.gateway => None, None => old.capacity, }; @@ -108,7 +120,7 @@ pub async fn cmd_assign_role( } _ => { let capacity = match args.capacity { - Some(c) => Some(c), + Some(c) => Some(c.as_u64()), None if args.gateway => None, None => return Err(Error::Message( "Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())), @@ -117,7 +129,7 @@ pub async fn cmd_assign_role( zone: args .zone .clone() - .ok_or("Please specifiy a zone with the -z flag")?, + .ok_or("Please specify a zone with the -z flag")?, capacity, tags: args.tags.clone(), } @@ -126,12 +138,14 @@ pub async fn cmd_assign_role( layout .staging + .get_mut() + .roles .merge(&roles.update_mutator(added_node, NodeRoleV(Some(new_entry)))); } send_layout(rpc_cli, rpc_host, layout).await?; - println!("Role changes are staged but not yet commited."); + println!("Role changes are staged but not yet committed."); println!("Use `garage layout show` to view staged role changes,"); println!("and `garage layout apply` to enact staged changes."); Ok(()) @@ -144,19 +158,21 @@ pub async fn cmd_remove_role( ) -> Result<(), Error> { let mut layout = fetch_layout(rpc_cli, rpc_host).await?; - let mut roles = layout.roles.clone(); - roles.merge(&layout.staging); + let mut roles = layout.current().roles.clone(); + roles.merge(&layout.staging.get().roles); let deleted_node = find_matching_node(roles.items().iter().map(|(id, _, _)| *id), &args.node_id)?; layout .staging + .get_mut() + .roles .merge(&roles.update_mutator(deleted_node, NodeRoleV(None))); send_layout(rpc_cli, rpc_host, layout).await?; - println!("Role removal is staged but not yet commited."); + println!("Role removal is staged but not yet committed."); println!("Use `garage layout show` to view staged role changes,"); println!("and `garage layout apply` to enact staged changes."); Ok(()) @@ -166,40 +182,44 @@ pub async fn cmd_show_layout( rpc_cli: &Endpoint, rpc_host: NodeID, ) -> Result<(), Error> { - let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + let layout = fetch_layout(rpc_cli, rpc_host).await?; println!("==== CURRENT CLUSTER LAYOUT ===="); - if !print_cluster_layout(&layout) { - println!("No nodes currently have a role in the cluster."); - println!("See `garage status` to view available nodes."); - } + print_cluster_layout(layout.current(), "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes."); println!(); - println!("Current cluster layout version: {}", layout.version); + println!( + "Current cluster layout version: {}", + layout.current().version + ); - if print_staging_role_changes(&layout) { - layout.roles.merge(&layout.staging); - - println!(); - println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ===="); - if !print_cluster_layout(&layout) { - println!("No nodes have a role in the new layout."); - } - println!(); + let has_role_changes = print_staging_role_changes(&layout); + if has_role_changes { + let v = layout.current().version; + let res_apply = layout.apply_staged_changes(Some(v + 1)); // this will print the stats of what partitions // will move around when we apply - if layout.calculate_partition_assignation() { - println!("To enact the staged role changes, type:"); - println!(); - println!(" garage layout apply --version {}", layout.version + 1); - println!(); - println!( - "You can also revert all proposed changes with: garage layout revert --version {}", - layout.version + 1 - ); - } else { - println!("Not enough nodes have an assigned role to maintain enough copies of data."); - println!("This new layout cannot yet be applied."); + match res_apply { + Ok((layout, msg)) => { + println!(); + println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ===="); + print_cluster_layout(layout.current(), "No nodes have a role in the new layout."); + println!(); + + for line in msg.iter() { + println!("{}", line); + } + println!("To enact the staged role changes, type:"); + println!(); + println!(" garage layout apply --version {}", v + 1); + println!(); + println!("You can also revert all proposed changes with: garage layout revert"); + } + Err(e) => { + println!("Error while trying to compute the assignment: {}", e); + println!("This new layout cannot yet be applied."); + println!("You can also revert all proposed changes with: garage layout revert"); + } } } @@ -213,11 +233,14 @@ pub async fn cmd_apply_layout( ) -> Result<(), Error> { let layout = fetch_layout(rpc_cli, rpc_host).await?; - let layout = layout.apply_staged_changes(apply_opt.version)?; + let (layout, msg) = layout.apply_staged_changes(apply_opt.version)?; + for line in msg.iter() { + println!("{}", line); + } send_layout(rpc_cli, rpc_host, layout).await?; - println!("New cluster layout with updated role assignation has been applied in cluster."); + println!("New cluster layout with updated role assignment has been applied in cluster."); println!("Data will now be moved around between nodes accordingly."); Ok(()) @@ -228,9 +251,15 @@ pub async fn cmd_revert_layout( rpc_host: NodeID, revert_opt: RevertLayoutOpt, ) -> Result<(), Error> { + if !revert_opt.yes { + return Err(Error::Message( + "Please add the --yes flag to run the layout revert operation".into(), + )); + } + let layout = fetch_layout(rpc_cli, rpc_host).await?; - let layout = layout.revert_staged_changes(revert_opt.version)?; + let layout = layout.revert_staged_changes()?; send_layout(rpc_cli, rpc_host, layout).await?; @@ -238,25 +267,226 @@ pub async fn cmd_revert_layout( Ok(()) } +pub async fn cmd_config_layout( + rpc_cli: &Endpoint, + rpc_host: NodeID, + config_opt: ConfigLayoutOpt, +) -> Result<(), Error> { + let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + + let mut did_something = false; + match config_opt.redundancy { + None => (), + Some(r_str) => { + let r = r_str + .parse::() + .ok_or_message("invalid zone redundancy value")?; + if let ZoneRedundancy::AtLeast(r_int) = r { + if r_int > layout.current().replication_factor { + return Err(Error::Message(format!( + "The zone redundancy must be smaller or equal to the \ + replication factor ({}).", + layout.current().replication_factor + ))); + } else if r_int < 1 { + return Err(Error::Message( + "The zone redundancy must be at least 1.".into(), + )); + } + } + + layout + .staging + .get_mut() + .parameters + .update(LayoutParameters { zone_redundancy: r }); + println!("The zone redundancy parameter has been set to '{}'.", r); + did_something = true; + } + } + + if !did_something { + return Err(Error::Message( + "Please specify an action for `garage layout config`".into(), + )); + } + + send_layout(rpc_cli, rpc_host, layout).await?; + Ok(()) +} + +pub async fn cmd_layout_history( + rpc_cli: &Endpoint, + rpc_host: NodeID, +) -> Result<(), Error> { + let layout = fetch_layout(rpc_cli, rpc_host).await?; + let min_stored = layout.min_stored(); + + println!("==== LAYOUT HISTORY ===="); + let mut table = vec!["Version\tStatus\tStorage nodes\tGateway nodes".to_string()]; + for ver in layout + .versions + .iter() + .rev() + .chain(layout.old_versions.iter().rev()) + { + let status = if ver.version == layout.current().version { + "current" + } else if ver.version >= min_stored { + "draining" + } else { + "historical" + }; + table.push(format!( + "#{}\t{}\t{}\t{}", + ver.version, + status, + ver.roles + .items() + .iter() + .filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_some())) + .count(), + ver.roles + .items() + .iter() + .filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_none())) + .count(), + )); + } + format_table(table); + println!(); + + if layout.versions.len() > 1 { + println!("==== UPDATE TRACKERS ===="); + println!("Several layout versions are currently live in the cluster, and data is being migrated."); + println!( + "This is the internal data that Garage stores to know which nodes have what data." + ); + println!(); + let mut table = vec!["Node\tAck\tSync\tSync_ack".to_string()]; + let all_nodes = layout.get_all_nodes(); + for node in all_nodes.iter() { + table.push(format!( + "{:?}\t#{}\t#{}\t#{}", + node, + layout.update_trackers.ack_map.get(node, min_stored), + layout.update_trackers.sync_map.get(node, min_stored), + layout.update_trackers.sync_ack_map.get(node, min_stored), + )); + } + table[1..].sort(); + format_table(table); + + let min_ack = layout + .update_trackers + .ack_map + .min_among(&all_nodes, layout.min_stored()); + + println!(); + println!( + "If some nodes are not catching up to the latest layout version in the update trackers," + ); + println!("it might be because they are offline or unable to complete a sync successfully."); + if min_ack < layout.current().version { + println!( + "You may force progress using `garage layout skip-dead-nodes --version {}`", + layout.current().version + ); + } else { + println!( + "You may force progress using `garage layout skip-dead-nodes --version {} --allow-missing-data`.", + layout.current().version + ); + } + } else { + println!("Your cluster is currently in a stable state with a single live layout version."); + println!("No metadata migration is in progress. Note that the migration of data blocks is not tracked,"); + println!( + "so you might want to keep old nodes online until their data directories become empty." + ); + } + + Ok(()) +} + +pub async fn cmd_layout_skip_dead_nodes( + rpc_cli: &Endpoint, + rpc_host: NodeID, + opt: SkipDeadNodesOpt, +) -> Result<(), Error> { + let status = fetch_status(rpc_cli, rpc_host).await?; + let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + + if layout.versions.len() == 1 { + return Err(Error::Message( + "This command cannot be called when there is only one live cluster layout version" + .into(), + )); + } + + let min_v = layout.min_stored(); + if opt.version <= min_v || opt.version > layout.current().version { + return Err(Error::Message(format!( + "Invalid version, you may use the following version numbers: {}", + (min_v + 1..=layout.current().version) + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ))); + } + + let all_nodes = layout.get_all_nodes(); + let mut did_something = false; + for node in all_nodes.iter() { + // Update ACK tracker for dead nodes or for all nodes if --allow-missing-data + if opt.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) { + if layout.update_trackers.ack_map.set_max(*node, opt.version) { + println!("Increased the ACK tracker for node {:?}", node); + did_something = true; + } + } + + // If --allow-missing-data, update SYNC tracker for all nodes. + if opt.allow_missing_data { + if layout.update_trackers.sync_map.set_max(*node, opt.version) { + println!("Increased the SYNC tracker for node {:?}", node); + did_something = true; + } + } + } + + if did_something { + send_layout(rpc_cli, rpc_host, layout).await?; + println!("Success."); + Ok(()) + } else if !opt.allow_missing_data { + Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into())) + } else { + Err(Error::Message( + "Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(), + )) + } +} + // --- utility --- pub async fn fetch_layout( rpc_cli: &Endpoint, rpc_host: NodeID, -) -> Result { +) -> Result { match rpc_cli .call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL) .await?? { SystemRpc::AdvertiseClusterLayout(t) => Ok(t), - resp => Err(Error::Message(format!("Invalid RPC response: {:?}", resp))), + resp => Err(Error::unexpected_rpc_message(resp)), } } pub async fn send_layout( rpc_cli: &Endpoint, rpc_host: NodeID, - layout: ClusterLayout, + layout: LayoutHistory, ) -> Result<(), Error> { rpc_cli .call( @@ -268,59 +498,85 @@ pub async fn send_layout( Ok(()) } -pub fn print_cluster_layout(layout: &ClusterLayout) -> bool { - let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()]; +pub fn print_cluster_layout(layout: &LayoutVersion, empty_msg: &str) { + let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()]; for (id, _, role) in layout.roles.items().iter() { let role = match &role.0 { Some(r) => r, _ => continue, }; let tags = role.tags.join(","); - table.push(format!( - "{:?}\t{}\t{}\t{}", - id, - tags, - role.zone, - role.capacity_string() - )); + let usage = layout.get_node_usage(id).unwrap_or(0); + let capacity = layout.get_node_capacity(id).unwrap_or(0); + if capacity > 0 { + table.push(format!( + "{:?}\t{}\t{}\t{}\t{} ({:.1}%)", + id, + tags, + role.zone, + role.capacity_string(), + ByteSize::b(usage as u64 * layout.partition_size).to_string_as(false), + (100.0 * usage as f32 * layout.partition_size as f32) / (capacity as f32) + )); + } else { + table.push(format!( + "{:?}\t{}\t{}\t{}", + id, + tags, + role.zone, + role.capacity_string() + )); + }; } - if table.len() == 1 { - false - } else { + if table.len() > 1 { format_table(table); - true + println!(); + println!("Zone redundancy: {}", layout.parameters.zone_redundancy); + } else { + println!("{}", empty_msg); } } -pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool { - let has_changes = layout - .staging +pub fn print_staging_role_changes(layout: &LayoutHistory) -> bool { + let staging = layout.staging.get(); + let has_role_changes = staging + .roles .items() .iter() - .any(|(k, _, v)| layout.roles.get(k) != Some(v)); + .any(|(k, _, v)| layout.current().roles.get(k) != Some(v)); + let has_layout_changes = *staging.parameters.get() != layout.current().parameters; - if has_changes { + if has_role_changes || has_layout_changes { println!(); println!("==== STAGED ROLE CHANGES ===="); - let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()]; - for (id, _, role) in layout.staging.items().iter() { - if layout.roles.get(id) == Some(role) { - continue; - } - if let Some(role) = &role.0 { - let tags = role.tags.join(","); - table.push(format!( - "{:?}\t{}\t{}\t{}", - id, - tags, - role.zone, - role.capacity_string() - )); - } else { - table.push(format!("{:?}\tREMOVED", id)); + if has_role_changes { + let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()]; + for (id, _, role) in staging.roles.items().iter() { + if layout.current().roles.get(id) == Some(role) { + continue; + } + if let Some(role) = &role.0 { + let tags = role.tags.join(","); + table.push(format!( + "{:?}\t{}\t{}\t{}", + id, + tags, + role.zone, + role.capacity_string() + )); + } else { + table.push(format!("{:?}\tREMOVED", id)); + } } + format_table(table); + println!(); + } + if has_layout_changes { + println!( + "Zone redundancy: {}", + staging.parameters.get().zone_redundancy + ); } - format_table(table); true } else { false diff --git a/src/garage/cli/mod.rs b/src/garage/cli/mod.rs index 17a2d8ce..e131f62c 100644 --- a/src/garage/cli/mod.rs +++ b/src/garage/cli/mod.rs @@ -4,6 +4,8 @@ pub(crate) mod layout; pub(crate) mod structs; pub(crate) mod util; +pub(crate) mod convert_db; + pub(crate) use cmd::*; pub(crate) use init::*; pub(crate) use layout::*; diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 01ae92da..386a213b 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -3,6 +3,8 @@ use structopt::StructOpt; use garage_util::version::garage_version; +use crate::cli::convert_db; + #[derive(StructOpt, Debug)] pub enum Command { /// Run Garage server @@ -17,7 +19,7 @@ pub enum Command { #[structopt(name = "node", version = garage_version())] Node(NodeOperation), - /// Operations on the assignation of node roles in the cluster layout + /// Operations on the assignment of node roles in the cluster layout #[structopt(name = "layout", version = garage_version())] Layout(LayoutOperation), @@ -29,11 +31,6 @@ pub enum Command { #[structopt(name = "key", version = garage_version())] Key(KeyOperation), - /// Run migrations from previous Garage version - /// (DO NOT USE WITHOUT READING FULL DOCUMENTATION) - #[structopt(name = "migrate", version = garage_version())] - Migrate(MigrateOpt), - /// Start repair of node data on remote node #[structopt(name = "repair", version = garage_version())] Repair(RepairOpt), @@ -51,14 +48,23 @@ pub enum Command { #[structopt(name = "worker", version = garage_version())] Worker(WorkerOperation), - /// Low-level debug operations on data blocks + /// Low-level node-local debug operations on data blocks #[structopt(name = "block", version = garage_version())] Block(BlockOperation), + + /// Operations on the metadata db + #[structopt(name = "meta", version = garage_version())] + Meta(MetaOperation), + + /// Convert metadata db between database engine formats + #[structopt(name = "convert-db", version = garage_version())] + ConvertDb(convert_db::ConvertDbOpt), } #[derive(StructOpt, Debug)] pub enum NodeOperation { - /// Print identifier (public key) of this Garage node + /// Print the full node ID (public key) of this Garage node, and its publicly reachable IP + /// address and port if they are specified in config file under `rpc_public_addr` #[structopt(name = "id", version = garage_version())] NodeId(NodeIdOpt), @@ -76,8 +82,9 @@ pub struct NodeIdOpt { #[derive(StructOpt, Debug)] pub struct ConnectNodeOpt { - /// Node public key and address, in the format: - /// `@:` + /// Full node ID (public key) and IP address and port, in the format: + /// `@:`. + /// You can retrieve this information on the target node using `garage node id`. pub(crate) node: String, } @@ -91,6 +98,10 @@ pub enum LayoutOperation { #[structopt(name = "remove", version = garage_version())] Remove(RemoveRoleOpt), + /// Configure parameters value for the layout computation + #[structopt(name = "config", version = garage_version())] + Config(ConfigLayoutOpt), + /// Show roles currently assigned to nodes and changes staged for commit #[structopt(name = "show", version = garage_version())] Show, @@ -102,6 +113,14 @@ pub enum LayoutOperation { /// Revert staged changes to cluster layout #[structopt(name = "revert", version = garage_version())] Revert(RevertLayoutOpt), + + /// View the history of layouts in the cluster + #[structopt(name = "history", version = garage_version())] + History, + + /// Skip dead nodes when awaiting for a new layout version to be synchronized + #[structopt(name = "skip-dead-nodes", version = garage_version())] + SkipDeadNodes(SkipDeadNodesOpt), } #[derive(StructOpt, Debug)] @@ -114,9 +133,9 @@ pub struct AssignRoleOpt { #[structopt(short = "z", long = "zone")] pub(crate) zone: Option, - /// Capacity (in relative terms, use 1 to represent your smallest server) + /// Storage capacity, in bytes (supported suffixes: B, KB, MB, GB, TB, PB) #[structopt(short = "c", long = "capacity")] - pub(crate) capacity: Option, + pub(crate) capacity: Option, /// Gateway-only node #[structopt(short = "g", long = "gateway")] @@ -137,6 +156,13 @@ pub struct RemoveRoleOpt { pub(crate) node_id: String, } +#[derive(StructOpt, Debug)] +pub struct ConfigLayoutOpt { + /// Zone redundancy parameter ('none'/'max' or integer) + #[structopt(short = "r", long = "redundancy")] + pub(crate) redundancy: Option, +} + #[derive(StructOpt, Debug)] pub struct ApplyLayoutOpt { /// Version number of new configuration: this command will fail if @@ -147,9 +173,21 @@ pub struct ApplyLayoutOpt { #[derive(StructOpt, Debug)] pub struct RevertLayoutOpt { - /// Version number of old configuration to which to revert + /// The revert operation will not be ran unless this flag is added + #[structopt(long = "yes")] + pub(crate) yes: bool, +} + +#[derive(StructOpt, Debug)] +pub struct SkipDeadNodesOpt { + /// Version number of the layout to assume is currently up-to-date. + /// This will generally be the current layout version. #[structopt(long = "version")] - pub(crate) version: Option, + pub(crate) version: u64, + /// Allow the skip even if a quorum of nodes could not be found for + /// the data among the remaining nodes + #[structopt(long = "allow-missing-data")] + pub(crate) allow_missing_data: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug)] @@ -216,7 +254,7 @@ pub struct WebsiteOpt { #[structopt(short = "i", long = "index-document", default_value = "index.html")] pub index_document: String, - /// Error document: the optionnal document returned when an error occurs + /// Error document: the optional document returned when an error occurs #[structopt(short = "e", long = "error-document")] pub error_document: Option, } @@ -317,11 +355,11 @@ pub enum KeyOperation { /// Get key info #[structopt(name = "info", version = garage_version())] - Info(KeyOpt), + Info(KeyInfoOpt), /// Create new key - #[structopt(name = "new", version = garage_version())] - New(KeyNewOpt), + #[structopt(name = "create", version = garage_version())] + Create(KeyNewOpt), /// Rename key #[structopt(name = "rename", version = garage_version())] @@ -345,15 +383,18 @@ pub enum KeyOperation { } #[derive(Serialize, Deserialize, StructOpt, Debug)] -pub struct KeyOpt { +pub struct KeyInfoOpt { /// ID or name of the key pub key_pattern: String, + /// Whether to display the secret key + #[structopt(long = "show-secret")] + pub show_secret: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug)] pub struct KeyNewOpt { /// Name of the key - #[structopt(long = "name", default_value = "Unnamed key")] + #[structopt(default_value = "Unnamed key")] pub name: String, } @@ -397,23 +438,10 @@ pub struct KeyImportOpt { /// Key name #[structopt(short = "n", default_value = "Imported key")] pub name: String, -} -#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] -pub struct MigrateOpt { - /// Confirm the launch of the migrate operation + /// Confirm key import #[structopt(long = "yes")] pub yes: bool, - - #[structopt(subcommand)] - pub what: MigrateWhat, -} - -#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] -pub enum MigrateWhat { - /// Migrate buckets and permissions from v0.5.0 - #[structopt(name = "buckets050", version = garage_version())] - Buckets050, } #[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] @@ -432,24 +460,40 @@ pub struct RepairOpt { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum RepairWhat { - /// Only do a full sync of metadata tables + /// Do a full sync of metadata tables #[structopt(name = "tables", version = garage_version())] Tables, - /// Only repair (resync/rebalance) the set of stored blocks + /// Repair (resync/rebalance) the set of stored blocks in the cluster #[structopt(name = "blocks", version = garage_version())] Blocks, - /// Only redo the propagation of object deletions to the version table (slow) + /// Clear the block resync queue. The list of blocks in errored state + /// is cleared as well. You MUST run `garage repair blocks` after invoking this. + #[structopt(name = "clear-resync-queue", version = garage_version())] + ClearResyncQueue, + /// Repropagate object deletions to the version table #[structopt(name = "versions", version = garage_version())] Versions, - /// Only redo the propagation of version deletions to the block ref table (extremely slow) - #[structopt(name = "block_refs", version = garage_version())] + /// Repropagate object deletions to the multipart upload table + #[structopt(name = "mpu", version = garage_version())] + MultipartUploads, + /// Repropagate version deletions to the block ref table + #[structopt(name = "block-refs", version = garage_version())] BlockRefs, - /// Verify integrity of all blocks on disc (extremely slow, i/o intensive) + /// Recalculate block reference counters + #[structopt(name = "block-rc", version = garage_version())] + BlockRc, + /// Fix inconsistency in bucket aliases (WARNING: EXPERIMENTAL) + #[structopt(name = "aliases", version = garage_version())] + Aliases, + /// Verify integrity of all blocks on disc #[structopt(name = "scrub", version = garage_version())] Scrub { #[structopt(subcommand)] cmd: ScrubCmd, }, + /// Rebalance data blocks among HDDs on individual nodes + #[structopt(name = "rebalance", version = garage_version())] + Rebalance, } #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] @@ -501,10 +545,6 @@ pub struct StatsOpt { #[structopt(short = "a", long = "all-nodes")] pub all_nodes: bool, - /// Gather detailed statistics (this can be long) - #[structopt(short = "d", long = "detailed")] - pub detailed: bool, - /// Don't show global cluster stats (internal use in RPC) #[structopt(skip)] #[serde(default)] @@ -585,3 +625,14 @@ pub enum BlockOperation { blocks: Vec, }, } + +#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)] +pub enum MetaOperation { + /// Save a snapshot of the metadata db file + #[structopt(name = "snapshot", version = garage_version())] + Snapshot { + /// Run on all nodes instead of only local node + #[structopt(long = "all")] + all: bool, + }, +} diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs index 2c6be2f4..21c14f42 100644 --- a/src/garage/cli/util.rs +++ b/src/garage/cli/util.rs @@ -1,19 +1,20 @@ use std::collections::HashMap; use std::time::Duration; +use format_table::format_table; use garage_util::background::*; use garage_util::crdt::*; use garage_util::data::*; use garage_util::error::*; -use garage_util::formater::format_table; use garage_util::time::*; use garage_block::manager::BlockResyncErrorInfo; use garage_model::bucket_table::*; use garage_model::key_table::*; -use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS}; -use garage_model::s3::version_table::Version; +use garage_model::s3::mpu_table::{self, MultipartUpload}; +use garage_model::s3::object_table; +use garage_model::s3::version_table::*; use crate::cli::structs::WorkerListOpt; @@ -135,6 +136,7 @@ pub fn print_bucket_info( bucket: &Bucket, relevant_keys: &HashMap, counters: &HashMap, + mpu_counters: &HashMap, ) { let key_name = |k| { relevant_keys @@ -148,7 +150,7 @@ pub fn print_bucket_info( Deletable::Deleted => println!("Bucket is deleted."), Deletable::Present(p) => { let size = - bytesize::ByteSize::b(counters.get(BYTES).cloned().unwrap_or_default() as u64); + bytesize::ByteSize::b(*counters.get(object_table::BYTES).unwrap_or(&0) as u64); println!( "\nSize: {} ({})", size.to_string_as(true), @@ -156,14 +158,22 @@ pub fn print_bucket_info( ); println!( "Objects: {}", - counters.get(OBJECTS).cloned().unwrap_or_default() + *counters.get(object_table::OBJECTS).unwrap_or(&0) + ); + println!( + "Unfinished uploads (multipart and non-multipart): {}", + *counters.get(object_table::UNFINISHED_UPLOADS).unwrap_or(&0) ); println!( "Unfinished multipart uploads: {}", - counters - .get(UNFINISHED_UPLOADS) - .cloned() - .unwrap_or_default() + *mpu_counters.get(mpu_table::UPLOADS).unwrap_or(&0) + ); + let mpu_size = + bytesize::ByteSize::b(*mpu_counters.get(mpu_table::BYTES).unwrap_or(&0) as u64); + println!( + "Size of unfinished multipart uploads: {} ({})", + mpu_size.to_string_as(true), + mpu_size.to_string_as(false), ); println!("\nWebsite access: {}", p.website_config.get().is_some()); @@ -373,41 +383,66 @@ pub fn print_block_error_list(el: Vec) { let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()]; for e in el { + let next_try = if e.next_try > now { + tf2.convert(Duration::from_millis(e.next_try - now)) + } else { + "asap".to_string() + }; table.push(format!( "{}\t{}\t{}\t{}\tin {}", hex::encode(e.hash.as_slice()), e.refcount, e.error_count, tf.convert(Duration::from_millis(now - e.last_try)), - tf2.convert(Duration::from_millis(e.next_try - now)) + next_try )); } format_table(table); } -pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec>) { +pub fn print_block_info( + hash: Hash, + refcount: u64, + versions: Vec>, + uploads: Vec, +) { println!("Block hash: {}", hex::encode(hash.as_slice())); println!("Refcount: {}", refcount); println!(); - let mut table = vec!["Version\tBucket\tKey\tDeleted".into()]; + let mut table = vec!["Version\tBucket\tKey\tMPU\tDeleted".into()]; let mut nondeleted_count = 0; for v in versions.iter() { match v { Ok(ver) => { - table.push(format!( - "{:?}\t{:?}\t{}\t{:?}", - ver.uuid, - ver.bucket_id, - ver.key, - ver.deleted.get() - )); + match &ver.backlink { + VersionBacklink::Object { bucket_id, key } => { + table.push(format!( + "{:?}\t{:?}\t{}\t\t{:?}", + ver.uuid, + bucket_id, + key, + ver.deleted.get() + )); + } + VersionBacklink::MultipartUpload { upload_id } => { + let upload = uploads.iter().find(|x| x.upload_id == *upload_id); + table.push(format!( + "{:?}\t{:?}\t{}\t{:?}\t{:?}", + ver.uuid, + upload.map(|u| u.bucket_id).unwrap_or_default(), + upload.map(|u| u.key.as_str()).unwrap_or_default(), + upload_id, + ver.deleted.get() + )); + } + } if !ver.deleted.get() { nondeleted_count += 1; } } Err(vh) => { - table.push(format!("{:?}\t\t\tyes", vh)); + table.push(format!("{:?}\t\t\t\tyes", vh)); } } } @@ -415,6 +450,8 @@ pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec@: + /// Host to connect to for admin operations, in the format: @: #[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")] pub rpc_host: Option, @@ -63,32 +66,12 @@ struct Opt { cmd: Command, } -#[derive(StructOpt, Debug)] -pub struct Secrets { - /// RPC secret network key, used to replace rpc_secret in config.toml when running the - /// daemon or doing admin operations - #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] - pub rpc_secret: Option, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] - pub admin_token: Option, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] - pub metrics_token: Option, -} - #[tokio::main] async fn main() { // Initialize version and features info let features = &[ #[cfg(feature = "k2v")] "k2v", - #[cfg(feature = "sled")] - "sled", #[cfg(feature = "lmdb")] "lmdb", #[cfg(feature = "sqlite")] @@ -108,6 +91,12 @@ async fn main() { ][..]; if let Some(git_version) = option_env!("GIT_VERSION") { garage_util::version::init_version(git_version); + } else { + garage_util::version::init_version(git_version::git_version!( + prefix = "git:", + cargo_prefix = "cargo:", + fallback = "unknown" + )); } garage_util::version::init_features(features); @@ -118,7 +107,7 @@ async fn main() { ); // Initialize panic handler that aborts on panic and shows a nice message. - // By default, Tokio continues runing normally when a task panics. We want + // By default, Tokio continues running normally when a task panics. We want // to avoid this behavior in Garage as this would risk putting the process in an // unknown/uncontrollable state. We prefer to exit the process and restart it // from scratch, so that it boots back into a fresh, known state. @@ -149,17 +138,8 @@ async fn main() { let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); // Initialize logging as well as other libraries used in Garage - if std::env::var("RUST_LOG").is_err() { - let default_log = match &opt.cmd { - Command::Server => "netapp=info,garage=info", - _ => "netapp=warn,garage=warn", - }; - std::env::set_var("RUST_LOG", default_log) - } - tracing_subscriber::fmt() - .with_writer(std::io::stderr) - .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) - .init(); + init_logging(&opt); + sodiumoxide::init().expect("Unable to init sodiumoxide"); let res = match opt.cmd { @@ -167,6 +147,9 @@ async fn main() { Command::OfflineRepair(repair_opt) => { repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt).await } + Command::ConvertDb(conv_opt) => { + cli::convert_db::do_conversion(conv_opt).map_err(From::from) + } Command::Node(NodeOperation::NodeId(node_id_opt)) => { node_id_command(opt.config_file, node_id_opt.quiet) } @@ -179,8 +162,99 @@ async fn main() { } } +fn init_logging(opt: &Opt) { + if std::env::var("RUST_LOG").is_err() { + let default_log = match &opt.cmd { + Command::Server => "netapp=info,garage=info", + _ => "netapp=warn,garage=warn", + }; + std::env::set_var("RUST_LOG", default_log) + } + + let env_filter = tracing_subscriber::filter::EnvFilter::from_default_env(); + + if std::env::var("GARAGE_LOG_TO_SYSLOG") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "syslog")] + { + use std::ffi::CStr; + use syslog_tracing::{Facility, Options, Syslog}; + + let syslog = Syslog::new( + CStr::from_bytes_with_nul(b"garage\0").unwrap(), + Options::LOG_PID | Options::LOG_PERROR, + Facility::Daemon, + ) + .expect("Unable to init syslog"); + + tracing_subscriber::fmt() + .with_writer(syslog) + .with_env_filter(env_filter) + .with_ansi(false) // disable ANSI escape sequences (colours) + .with_file(false) + .with_level(false) + .without_time() + .compact() + .init(); + + return; + } + #[cfg(not(feature = "syslog"))] + { + eprintln!("Syslog support is not enabled in this build."); + std::process::exit(1); + } + } + + if std::env::var("GARAGE_LOG_TO_JOURNALD") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "journald")] + { + use tracing_journald::{Priority, PriorityMappings}; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + + let registry = tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::sink)) + .with(env_filter); + match tracing_journald::layer() { + Ok(layer) => { + registry + .with(layer.with_priority_mappings(PriorityMappings { + info: Priority::Informational, + debug: Priority::Debug, + ..PriorityMappings::new() + })) + .init(); + } + Err(e) => { + eprintln!("Couldn't connect to journald: {}.", e); + std::process::exit(1); + } + } + return; + } + #[cfg(not(feature = "journald"))] + { + eprintln!("Journald support is not enabled in this build."); + std::process::exit(1); + } + } + + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(env_filter) + .init(); +} + async fn cli_command(opt: Opt) -> Result<(), Error> { - let config = if opt.secrets.rpc_secret.is_none() || opt.rpc_host.is_none() { + let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none()) + || opt.rpc_host.is_none() + { Some(garage_util::config::read_config(opt.config_file.clone()) .err_context(format!("Unable to read configuration file {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy()))?) } else { @@ -188,25 +262,30 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { }; // Find and parse network RPC secret - let net_key_hex_str = opt - .secrets - .rpc_secret - .as_ref() - .or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref())) - .ok_or("No RPC secret provided")?; + let mut rpc_secret = config.as_ref().and_then(|c| c.rpc_secret.clone()); + secrets::fill_secret( + &mut rpc_secret, + &config.as_ref().and_then(|c| c.rpc_secret_file.clone()), + &opt.secrets.rpc_secret, + &opt.secrets.rpc_secret_file, + "rpc_secret", + true, + )?; + + let net_key_hex_str = rpc_secret.ok_or("No RPC secret provided")?; let network_key = NetworkKey::from_slice( - &hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], + &hex::decode(&net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], ) .ok_or("Invalid RPC secret provided (wrong length)")?; // Generate a temporary keypair for our RPC client let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); - let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None); // Find and parse the address of the target host let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host { - let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; + let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; (id, addrs[0], false) } else { let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir) @@ -236,7 +315,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { addr ); } - Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?; + Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?; } let system_rpc_endpoint = netapp.endpoint::(SYSTEM_RPC_PATH.into()); @@ -249,16 +328,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { Ok(x) => Ok(x), } } - -fn fill_secrets(mut config: Config, secrets: Secrets) -> Config { - if secrets.rpc_secret.is_some() { - config.rpc_secret = secrets.rpc_secret; - } - if secrets.admin_token.is_some() { - config.admin.admin_token = secrets.admin_token; - } - if secrets.metrics_token.is_some() { - config.admin.metrics_token = secrets.metrics_token; - } - config -} diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs index f4edcf03..45024e71 100644 --- a/src/garage/repair/offline.rs +++ b/src/garage/repair/offline.rs @@ -6,7 +6,7 @@ use garage_util::error::*; use garage_model::garage::Garage; use crate::cli::structs::*; -use crate::{fill_secrets, Secrets}; +use crate::secrets::{fill_secrets, Secrets}; pub async fn offline_repair( config_file: PathBuf, @@ -20,7 +20,7 @@ pub async fn offline_repair( } info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; info!("Initializing Garage main data store..."); let garage = Garage::new(config)?; diff --git a/src/garage/repair/online.rs b/src/garage/repair/online.rs index 0e14ed51..6a7dafcf 100644 --- a/src/garage/repair/online.rs +++ b/src/garage/repair/online.rs @@ -1,21 +1,31 @@ +use std::future::Future; use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; use tokio::sync::watch; +use garage_block::manager::BlockManager; use garage_block::repair::ScrubWorkerCommand; + use garage_model::garage::Garage; use garage_model::s3::block_ref_table::*; +use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; + +use garage_table::replication::*; use garage_table::*; + use garage_util::background::*; +use garage_util::data::*; use garage_util::error::Error; use garage_util::migrate::Migrate; use crate::*; +const RC_REPAIR_ITER_COUNT: usize = 64; + pub async fn launch_online_repair( garage: &Arc, bg: &BackgroundRunner, @@ -32,11 +42,22 @@ pub async fn launch_online_repair( } RepairWhat::Versions => { info!("Repairing the versions table"); - bg.spawn_worker(RepairVersionsWorker::new(garage.clone())); + bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions)); + } + RepairWhat::MultipartUploads => { + info!("Repairing the multipart uploads table"); + bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu)); } RepairWhat::BlockRefs => { info!("Repairing the block refs table"); - bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone())); + bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs)); + } + RepairWhat::BlockRc => { + info!("Repairing the block reference counters"); + bg.spawn_worker(BlockRcRepair::new( + garage.block_manager.clone(), + garage.block_ref_table.clone(), + )); } RepairWhat::Blocks => { info!("Repairing the stored blocks"); @@ -61,76 +82,90 @@ pub async fn launch_online_repair( info!("Sending command to scrub worker: {:?}", cmd); garage.block_manager.send_scrub_command(cmd).await?; } + RepairWhat::Rebalance => { + info!("Rebalancing the stored blocks among storage locations"); + bg.spawn_worker(garage_block::repair::RebalanceWorker::new( + garage.block_manager.clone(), + )); + } + RepairWhat::Aliases => { + info!("Repairing bucket aliases (foreground)"); + garage.locked_helper().await.repair_aliases().await?; + } + RepairWhat::ClearResyncQueue => { + let garage = garage.clone(); + tokio::task::spawn_blocking(move || garage.block_manager.resync.clear_resync_queue()) + .await?? + } } Ok(()) } // ---- -struct RepairVersionsWorker { +trait TableRepair: Send + Sync + 'static { + type T: TableSchema; + + fn table(garage: &Garage) -> &Table; + + fn process( + &mut self, + garage: &Garage, + entry: <::T as TableSchema>::E, + ) -> impl Future> + Send; +} + +struct TableRepairWorker { garage: Arc, pos: Vec, counter: usize, + repairs: usize, + inner: T, } -impl RepairVersionsWorker { - fn new(garage: Arc) -> Self { +impl TableRepairWorker { + fn new(garage: Arc, inner: R) -> Self { Self { garage, + inner, pos: vec![], counter: 0, + repairs: 0, } } } #[async_trait] -impl Worker for RepairVersionsWorker { +impl Worker for TableRepairWorker { fn name(&self) -> String { - "Version repair worker".into() + format!("{} repair worker", R::T::TABLE_NAME) } fn status(&self) -> WorkerStatus { WorkerStatus { - progress: Some(self.counter.to_string()), + progress: Some(format!("{} ({})", self.counter, self.repairs)), ..Default::default() } } async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { - let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? { + let (item_bytes, next_pos) = match R::table(&self.garage).data.store.get_gt(&self.pos)? { Some((k, v)) => (v, k), None => { - info!("repair_versions: finished, done {}", self.counter); + info!( + "{}: finished, done {}, fixed {}", + self.name(), + self.counter, + self.repairs + ); return Ok(WorkerState::Done); } }; - let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?; - if !version.deleted.get() { - let object = self - .garage - .object_table - .get(&version.bucket_id, &version.key) - .await?; - let version_exists = match object { - Some(o) => o - .versions() - .iter() - .any(|x| x.uuid == version.uuid && x.state != ObjectVersionState::Aborted), - None => false, - }; - if !version_exists { - info!("Repair versions: marking version as deleted: {:?}", version); - self.garage - .version_table - .insert(&Version::new( - version.uuid, - version.bucket_id, - version.key, - true, - )) - .await?; - } + let entry = ::E::decode(&item_bytes) + .ok_or_message("Cannot decode table entry")?; + if self.inner.process(&self.garage, entry).await? { + self.repairs += 1; } self.counter += 1; @@ -146,72 +181,211 @@ impl Worker for RepairVersionsWorker { // ---- -struct RepairBlockrefsWorker { - garage: Arc, - pos: Vec, - counter: usize, -} +struct RepairVersions; -impl RepairBlockrefsWorker { - fn new(garage: Arc) -> Self { - Self { - garage, - pos: vec![], - counter: 0, - } - } -} +impl TableRepair for RepairVersions { + type T = VersionTable; -#[async_trait] -impl Worker for RepairBlockrefsWorker { - fn name(&self) -> String { - "Block refs repair worker".into() + fn table(garage: &Garage) -> &Table { + &garage.version_table } - fn status(&self) -> WorkerStatus { - WorkerStatus { - progress: Some(self.counter.to_string()), - ..Default::default() - } - } - - async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { - let (item_bytes, next_pos) = - match self.garage.block_ref_table.data.store.get_gt(&self.pos)? { - Some((k, v)) => (v, k), - None => { - info!("repair_block_ref: finished, done {}", self.counter); - return Ok(WorkerState::Done); - } + async fn process(&mut self, garage: &Garage, version: Version) -> Result { + if !version.deleted.get() { + let ref_exists = match &version.backlink { + VersionBacklink::Object { bucket_id, key } => garage + .object_table + .get(bucket_id, key) + .await? + .map(|o| { + o.versions().iter().any(|x| { + x.uuid == version.uuid && x.state != ObjectVersionState::Aborted + }) + }) + .unwrap_or(false), + VersionBacklink::MultipartUpload { upload_id } => garage + .mpu_table + .get(upload_id, &EmptyKey) + .await? + .map(|u| !u.deleted.get()) + .unwrap_or(false), }; - let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?; + if !ref_exists { + info!("Repair versions: marking version as deleted: {:?}", version); + garage + .version_table + .insert(&Version::new(version.uuid, version.backlink, true)) + .await?; + return Ok(true); + } + } + + Ok(false) + } +} + +// ---- + +struct RepairBlockRefs; + +impl TableRepair for RepairBlockRefs { + type T = BlockRefTable; + + fn table(garage: &Garage) -> &Table { + &garage.block_ref_table + } + + async fn process(&mut self, garage: &Garage, mut block_ref: BlockRef) -> Result { if !block_ref.deleted.get() { - let version = self - .garage + let ref_exists = garage .version_table .get(&block_ref.version, &EmptyKey) - .await?; - // The version might not exist if it has been GC'ed - let ref_exists = version.map(|v| !v.deleted.get()).unwrap_or(false); + .await? + .map(|v| !v.deleted.get()) + .unwrap_or(false); + if !ref_exists { info!( "Repair block ref: marking block_ref as deleted: {:?}", block_ref ); - self.garage - .block_ref_table - .insert(&BlockRef { - block: block_ref.block, - version: block_ref.version, - deleted: true.into(), - }) - .await?; + block_ref.deleted.set(); + garage.block_ref_table.insert(&block_ref).await?; + return Ok(true); } } - self.counter += 1; - self.pos = next_pos; + Ok(false) + } +} + +// ---- + +struct RepairMpu; + +impl TableRepair for RepairMpu { + type T = MultipartUploadTable; + + fn table(garage: &Garage) -> &Table { + &garage.mpu_table + } + + async fn process(&mut self, garage: &Garage, mut mpu: MultipartUpload) -> Result { + if !mpu.deleted.get() { + let ref_exists = garage + .object_table + .get(&mpu.bucket_id, &mpu.key) + .await? + .map(|o| { + o.versions() + .iter() + .any(|x| x.uuid == mpu.upload_id && x.is_uploading(Some(true))) + }) + .unwrap_or(false); + + if !ref_exists { + info!( + "Repair multipart uploads: marking mpu as deleted: {:?}", + mpu + ); + mpu.parts.clear(); + mpu.deleted.set(); + garage.mpu_table.insert(&mpu).await?; + return Ok(true); + } + } + + Ok(false) + } +} + +// ===== block reference counter repair ===== + +pub struct BlockRcRepair { + block_manager: Arc, + block_ref_table: Arc>, + cursor: Hash, + counter: u64, + repairs: u64, +} + +impl BlockRcRepair { + fn new( + block_manager: Arc, + block_ref_table: Arc>, + ) -> Self { + Self { + block_manager, + block_ref_table, + cursor: [0u8; 32].into(), + counter: 0, + repairs: 0, + } + } +} + +#[async_trait] +impl Worker for BlockRcRepair { + fn name(&self) -> String { + format!("Block refcount repair worker") + } + + fn status(&self) -> WorkerStatus { + WorkerStatus { + progress: Some(format!("{} ({})", self.counter, self.repairs)), + ..Default::default() + } + } + + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + for _i in 0..RC_REPAIR_ITER_COUNT { + let next1 = self + .block_manager + .rc + .rc_table + .range(self.cursor.as_slice()..)? + .next() + .transpose()? + .map(|(k, _)| Hash::try_from(k.as_slice()).unwrap()); + let next2 = self + .block_ref_table + .data + .store + .range(self.cursor.as_slice()..)? + .next() + .transpose()? + .map(|(k, _)| Hash::try_from(&k[..32]).unwrap()); + let next = match (next1, next2) { + (Some(k1), Some(k2)) => std::cmp::min(k1, k2), + (Some(k), None) | (None, Some(k)) => k, + (None, None) => { + info!( + "{}: finished, done {}, fixed {}", + self.name(), + self.counter, + self.repairs + ); + return Ok(WorkerState::Done); + } + }; + + if self.block_manager.rc.recalculate_rc(&next)?.1 { + self.repairs += 1; + } + self.counter += 1; + if let Some(next_incr) = next.increment() { + self.cursor = next_incr; + } else { + info!( + "{}: finished, done {}, fixed {}", + self.name(), + self.counter, + self.repairs + ); + return Ok(WorkerState::Done); + } + } Ok(WorkerState::Busy) } diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs new file mode 100644 index 00000000..17781efe --- /dev/null +++ b/src/garage/secrets.rs @@ -0,0 +1,320 @@ +use std::path::PathBuf; + +use structopt::StructOpt; + +use garage_util::config::Config; +use garage_util::error::Error; + +/// Structure for secret values or paths that are passed as CLI arguments or environment +/// variables, instead of in the config file. +#[derive(StructOpt, Debug, Default, Clone)] +pub struct Secrets { + /// Skip permission check on files containing secrets + #[cfg(unix)] + #[structopt( + long = "allow-world-readable-secrets", + env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS" + )] + pub allow_world_readable_secrets: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml when running the + /// daemon or doing admin operations + #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] + pub rpc_secret: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret + /// when running the daemon or doing admin operations + #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] + pub rpc_secret_file: Option, + + /// Admin API authentication token, replaces admin.admin_token in config.toml when + /// running the Garage daemon + #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] + pub admin_token: Option, + + /// Admin API authentication token file path, replaces admin.admin_token in config.toml + /// and admin-token when running the Garage daemon + #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] + pub admin_token_file: Option, + + /// Metrics API authentication token, replaces admin.metrics_token in config.toml when + /// running the Garage daemon + #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] + pub metrics_token: Option, + + /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml + /// and metrics-token when running the Garage daemon + #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] + pub metrics_token_file: Option, +} + +/// Single function to fill all secrets in the Config struct from their correct source (value +/// from config or CLI param or env variable or read from a file specified in config or CLI +/// param or env variable) +pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result { + let allow_world_readable = secrets + .allow_world_readable_secrets + .unwrap_or(config.allow_world_readable_secrets); + + fill_secret( + &mut config.rpc_secret, + &config.rpc_secret_file, + &secrets.rpc_secret, + &secrets.rpc_secret_file, + "rpc_secret", + allow_world_readable, + )?; + + fill_secret( + &mut config.admin.admin_token, + &config.admin.admin_token_file, + &secrets.admin_token, + &secrets.admin_token_file, + "admin.admin_token", + allow_world_readable, + )?; + fill_secret( + &mut config.admin.metrics_token, + &config.admin.metrics_token_file, + &secrets.metrics_token, + &secrets.metrics_token_file, + "admin.metrics_token", + allow_world_readable, + )?; + + Ok(config) +} + +pub(crate) fn fill_secret( + config_secret: &mut Option, + config_secret_file: &Option, + cli_secret: &Option, + cli_secret_file: &Option, + name: &'static str, + allow_world_readable: bool, +) -> Result<(), Error> { + let cli_value = match (&cli_secret, &cli_secret_file) { + (Some(_), Some(_)) => { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + (Some(secret), None) => Some(secret.to_string()), + (None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?), + (None, None) => None, + }; + + if let Some(val) = cli_value { + if config_secret.is_some() || config_secret_file.is_some() { + debug!("Overriding secret `{}` using value specified using CLI argument or environment variable.", name); + } + + *config_secret = Some(val); + } else if let Some(file_path) = &config_secret_file { + if config_secret.is_some() { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + + *config_secret = Some(read_secret_file(file_path, allow_world_readable)?); + } + + Ok(()) +} + +fn read_secret_file(file_path: &PathBuf, allow_world_readable: bool) -> Result { + if !allow_world_readable { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(file_path)?; + if metadata.mode() & 0o077 != 0 { + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path.display(), metadata.mode()).into()); + } + } + } + + let secret_buf = std::fs::read_to_string(file_path)?; + + // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. + // also editors sometimes add a trailing newline + Ok(String::from(secret_buf.trim_end())) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + + use garage_util::config::read_config; + use garage_util::error::Error; + + use super::*; + + #[test] + fn test_rpc_secret_file_works() -> Result<(), Error> { + let path_secret = mktemp::Temp::new_file()?; + let mut file_secret = File::create(path_secret.as_path())?; + writeln!(file_secret, "foo")?; + drop(file_secret); + + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + let path_secret_path = path_secret.as_path(); + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config); + + // Second configuration file, same as previous one + // except it allows world-readable secrets. + let path_config_allow_world_readable = mktemp::Temp::new_file()?; + let mut file_config_allow_world_readable = + File::create(path_config_allow_world_readable.as_path())?; + writeln!( + file_config_allow_world_readable, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + allow_world_readable_secrets = true + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config_allow_world_readable); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets(config, Secrets::default())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + + // ---- Check non world-readable secrets config ---- + #[cfg(unix)] + { + let secrets_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(true), + ..Default::default() + }; + let secrets_no_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(false), + ..Default::default() + }; + + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(path_secret_path)?; + let mut perm = metadata.permissions(); + perm.set_mode(0o660); + std::fs::set_permissions(path_secret_path, perm)?; + + // Config file that just specifies the path + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_err()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err()); + + // Config file that also specifies to allow world_readable_secrets + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err()); + } + + // ---- Check alternative secrets specified on CLI ---- + + let path_secret2 = mktemp::Temp::new_file()?; + let mut file_secret2 = File::create(path_secret2.as_path())?; + writeln!(file_secret2, "bar")?; + drop(file_secret2); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("baz")); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret_file: Some(path_secret2.clone()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("bar")); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + rpc_secret_file: Some(path_secret2.clone()), + ..Default::default() + } + ) + .is_err()); + + drop(path_secret); + drop(path_secret2); + drop(path_config); + drop(path_config_allow_world_readable); + + Ok(()) + } + + #[test] + fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_factor = 3 + rpc_bind_addr = "[::]:3901" + rpc_secret= "dummy" + rpc_secret_file = "dummy" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + let config = read_config(path_config.to_path_buf())?; + assert_eq!( + "only one of `rpc_secret` and `rpc_secret_file` can be set", + fill_secrets(config, Secrets::default()) + .unwrap_err() + .to_string() + ); + drop(path_config); + drop(file_config); + Ok(()) + } +} diff --git a/src/garage/server.rs b/src/garage/server.rs index 958089c6..b81ae334 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -6,18 +6,18 @@ use garage_util::background::*; use garage_util::config::*; use garage_util::error::Error; -use garage_api::admin::api_server::AdminApiServer; -use garage_api::s3::api_server::S3ApiServer; +use garage_api_admin::api_server::AdminApiServer; +use garage_api_s3::api_server::S3ApiServer; use garage_model::garage::Garage; use garage_web::WebServer; #[cfg(feature = "k2v")] -use garage_api::k2v::api_server::K2VApiServer; +use garage_api_k2v::api_server::K2VApiServer; use crate::admin::*; +use crate::secrets::{fill_secrets, Secrets}; #[cfg(feature = "telemetry-otlp")] use crate::tracing_setup::*; -use crate::{fill_secrets, Secrets}; async fn wait_from(mut chan: watch::Receiver) { while !*chan.borrow() { @@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver) { pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> { info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; // ---- Initialize Garage internals ---- #[cfg(feature = "metrics")] - let metrics_exporter = opentelemetry_prometheus::exporter().init(); + let metrics_exporter = opentelemetry_prometheus::exporter() + .with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99]) + .with_default_histogram_boundaries(vec![ + 0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1, + 0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60., + 70., 100., + ]) + .init(); info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone())?; @@ -44,7 +51,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone()); info!("Spawning Garage workers..."); - garage.spawn_workers(&background); + garage.spawn_workers(&background)?; if config.admin.trace_sink.is_some() { info!("Initialize tracing..."); @@ -79,9 +86,9 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er "S3 API", tokio::spawn(S3ApiServer::run( garage.clone(), - *s3_bind_addr, + s3_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -94,9 +101,9 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er "K2V API", tokio::spawn(K2VApiServer::run( garage.clone(), - config.k2v_api.as_ref().unwrap().api_bind_addr, + config.k2v_api.as_ref().unwrap().api_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -106,14 +113,10 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er if let Some(web_config) = &config.s3_web { info!("Initializing web server..."); + let web_server = WebServer::new(garage.clone(), &web_config); servers.push(( "Web", - tokio::spawn(WebServer::run( - garage.clone(), - web_config.bind_addr, - web_config.root_domain.clone(), - wait_from(watch_cancel.clone()), - )), + tokio::spawn(web_server.run(web_config.bind_addr.clone(), watch_cancel.clone())), )); } @@ -121,7 +124,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er info!("Launching Admin API server..."); servers.push(( "Admin", - tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))), + tokio::spawn(admin_server.run(admin_bind_addr.clone(), watch_cancel.clone())), )); } @@ -130,20 +133,27 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er warn!("This Garage version is built without the metrics feature"); } - // Stuff runs + if servers.is_empty() { + // Nothing runs except netapp (not in servers) + // Await shutdown signal before proceeding to shutting down netapp + wait_from(watch_cancel).await; + } else { + // Stuff runs - // When a cancel signal is sent, stuff stops + // When a cancel signal is sent, stuff stops - // Collect stuff - for (desc, join_handle) in servers { - if let Err(e) = join_handle.await? { - error!("{} server exited with error: {}", desc, e); - } else { - info!("{} server exited without error.", desc); + // Collect stuff + for (desc, join_handle) in servers { + if let Err(e) = join_handle.await? { + error!("{} server exited with error: {}", desc, e); + } else { + info!("{} server exited without error.", desc); + } } } // Remove RPC handlers for system to break reference cycles + info!("Deregistering RPC handlers for shutdown..."); garage.system.netapp.drop_all_handlers(); opentelemetry::global::shutdown_tracer_provider(); @@ -152,6 +162,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er info!("Netapp exited"); // Drop all references so that stuff can terminate properly + garage.system.cleanup(); drop(garage); // Await for all background tasks to end @@ -172,10 +183,21 @@ fn watch_shutdown_signal() -> watch::Receiver { let mut sigterm = signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler"); let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler"); - tokio::select! { - _ = sigint.recv() => info!("Received SIGINT, shutting down."), - _ = sigterm.recv() => info!("Received SIGTERM, shutting down."), - _ = sighup.recv() => info!("Received SIGHUP, shutting down."), + loop { + tokio::select! { + _ = sigint.recv() => { + info!("Received SIGINT, shutting down."); + break + } + _ = sigterm.recv() => { + info!("Received SIGTERM, shutting down."); + break + } + _ = sighup.recv() => { + info!("Received SIGHUP, reload not supported."); + continue + } + } } send_cancel.send(true).unwrap(); }); diff --git a/src/garage/tests/bucket.rs b/src/garage/tests/bucket.rs index 0dec3cfa..2dda7e6f 100644 --- a/src/garage/tests/bucket.rs +++ b/src/garage/tests/bucket.rs @@ -1,7 +1,6 @@ use crate::common; use crate::common::ext::CommandExt; -use aws_sdk_s3::model::BucketLocationConstraint; -use aws_sdk_s3::output::DeleteBucketOutput; +use aws_sdk_s3::operation::delete_bucket::DeleteBucketOutput; #[tokio::test] async fn test_bucket_all() { @@ -63,10 +62,7 @@ async fn test_bucket_all() { .await .unwrap(); - match r.location_constraint.unwrap() { - BucketLocationConstraint::Unknown(v) if v.as_str() == "garage-integ-test" => (), - _ => unreachable!("wrong region"), - } + assert_eq!(r.location_constraint.unwrap().as_str(), "garage-integ-test"); } { // (Stub) check GetVersioning diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs index e9d4849a..7a6612cb 100644 --- a/src/garage/tests/common/client.rs +++ b/src/garage/tests/common/client.rs @@ -1,15 +1,18 @@ -use aws_sdk_s3::{Client, Config, Credentials, Endpoint}; +use aws_sdk_s3::config::BehaviorVersion; +use aws_sdk_s3::config::Credentials; +use aws_sdk_s3::{Client, Config}; -use super::garage::{Instance, Key}; +use super::garage::Key; +use crate::common::garage::DEFAULT_PORT; -pub fn build_client(instance: &Instance, key: &Key) -> Client { +pub fn build_client(key: &Key) -> Client { let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test"); - let endpoint = Endpoint::immutable(instance.s3_uri()); let config = Config::builder() + .endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT)) .region(super::REGION) .credentials_provider(credentials) - .endpoint_resolver(endpoint) + .behavior_version(BehaviorVersion::v2024_03_28()) .build(); Client::from_conf(config) diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 609eda97..6a8eed38 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -1,15 +1,23 @@ #![allow(dead_code)] use std::collections::HashMap; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use chrono::{offset::Utc, DateTime}; use hmac::{Hmac, Mac}; -use hyper::client::HttpConnector; -use hyper::{Body, Client, Method, Request, Response, Uri}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::header::{ + HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, HOST, +}; +use hyper::{Method, Request, Response, Uri}; +use hyper_util::client::legacy::{connect::HttpConnector, Client}; +use hyper_util::rt::TokioExecutor; use super::garage::{Instance, Key}; -use garage_api::signature; +use garage_api_common::signature; + +pub type Body = FullBody; /// You should ever only use this to send requests AWS sdk won't send, /// like to reproduce behavior of unusual implementations found to be @@ -19,7 +27,7 @@ pub struct CustomRequester { key: Key, uri: Uri, service: &'static str, - client: Client, + client: Client, } impl CustomRequester { @@ -28,7 +36,7 @@ impl CustomRequester { key: key.clone(), uri: instance.s3_uri(), service: "s3", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -37,7 +45,7 @@ impl CustomRequester { key: key.clone(), uri: instance.k2v_uri(), service: "k2v", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -56,6 +64,10 @@ impl CustomRequester { vhost_style: false, } } + + pub fn client(&self) -> &Client { + &self.client + } } pub struct RequestBuilder<'a> { @@ -139,9 +151,9 @@ impl<'a> RequestBuilder<'a> { self } - pub async fn send(&mut self) -> hyper::Result> { + pub async fn send(&mut self) -> Result, String> { // TODO this is a bit incorrect in that path and query params should be url-encoded and - // aren't, but this is good enought for now. + // aren't, but this is good enough for now. let query = query_param_to_string(&self.query_params); let (host, path) = if self.vhost_style { @@ -168,53 +180,123 @@ impl<'a> RequestBuilder<'a> { .unwrap(); let streaming_signer = signer.clone(); - let mut all_headers = self.signed_headers.clone(); + let mut all_headers = self + .signed_headers + .iter() + .map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + }) + .collect::(); let date = now.format(signature::LONG_DATETIME).to_string(); - all_headers.insert("x-amz-date".to_owned(), date); - all_headers.insert("host".to_owned(), host); + all_headers.insert(signature::X_AMZ_DATE, HeaderValue::from_str(&date).unwrap()); + all_headers.insert(HOST, HeaderValue::from_str(&host).unwrap()); - let body_sha = match self.body_signature { + let body_sha = match &self.body_signature { BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(), BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)), - BodySignature::Streaming(size) => { - all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned()); + BodySignature::Streaming { chunk_size } => { all_headers.insert( - "x-amz-decoded-content-length".to_owned(), - self.body.len().to_string(), + CONTENT_ENCODING, + HeaderValue::from_str("aws-chunked").unwrap(), ); - // Get lenght of body by doing the conversion to a streaming body with an + all_headers.insert( + HeaderName::from_static("x-amz-decoded-content-length"), + HeaderValue::from_str(&self.body.len().to_string()).unwrap(), + ); + // Get length of body by doing the conversion to a streaming body with an // invalid signature (we don't know the seed) just to get its length. This - // is a pretty lazy and inefficient way to do it, but it's enought for test + // is a pretty lazy and inefficient way to do it, but it's enough for test // code. all_headers.insert( - "content-length".to_owned(), - to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") - .len() - .to_string(), + CONTENT_LENGTH, + to_streaming_body( + &self.body, + *chunk_size, + String::new(), + signer.clone(), + now, + "", + ) + .len() + .to_string() + .try_into() + .unwrap(), ); "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned() } + BodySignature::StreamingUnsignedTrailer { + chunk_size, + trailer_algorithm, + trailer_value, + } => { + all_headers.insert( + CONTENT_ENCODING, + HeaderValue::from_str("aws-chunked").unwrap(), + ); + all_headers.insert( + HeaderName::from_static("x-amz-decoded-content-length"), + HeaderValue::from_str(&self.body.len().to_string()).unwrap(), + ); + all_headers.insert( + HeaderName::from_static("x-amz-trailer"), + HeaderValue::from_str(&trailer_algorithm).unwrap(), + ); + + all_headers.insert( + CONTENT_LENGTH, + to_streaming_unsigned_trailer_body( + &self.body, + *chunk_size, + &trailer_algorithm, + &trailer_value, + ) + .len() + .to_string() + .try_into() + .unwrap(), + ); + + "STREAMING-UNSIGNED-PAYLOAD-TRAILER".to_owned() + } }; - all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); + all_headers.insert( + signature::X_AMZ_CONTENT_SHA256, + HeaderValue::from_str(&body_sha).unwrap(), + ); - let mut signed_headers = all_headers + let mut signed_headers = all_headers.keys().cloned().collect::>(); + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + let signed_headers_str = signed_headers .iter() - .map(|(k, _)| k.as_ref()) - .collect::>(); - signed_headers.sort(); - let signed_headers = signed_headers.join(";"); + .map(ToString::to_string) + .collect::>() + .join(";"); - all_headers.extend(self.unsigned_headers.clone()); + all_headers.extend(self.unsigned_headers.iter().map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + })); + + let uri = Uri::try_from(&uri).unwrap(); + let query = signature::payload::parse_query_map(&uri).unwrap(); let canonical_request = signature::payload::canonical_request( + self.service, &self.method, - &Uri::try_from(&uri).unwrap(), + uri.path(), + &query, &all_headers, &signed_headers, &body_sha, - ); + ) + .unwrap(); let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request); @@ -222,33 +304,72 @@ impl<'a> RequestBuilder<'a> { let signature = hex::encode(signer.finalize().into_bytes()); let authorization = format!( "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}", - self.requester.key.id, scope, signed_headers, signature + self.requester.key.id, scope, signed_headers_str, signature + ); + all_headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&authorization).unwrap(), ); - all_headers.insert("authorization".to_owned(), authorization); let mut request = Request::builder(); - for (k, v) in all_headers { - request = request.header(k, v); - } + *request.headers_mut().unwrap() = all_headers; - let body = if let BodySignature::Streaming(size) = self.body_signature { - to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope) - } else { - self.body.clone() + let body = match &self.body_signature { + BodySignature::Streaming { chunk_size } => to_streaming_body( + &self.body, + *chunk_size, + signature, + streaming_signer, + now, + &scope, + ), + BodySignature::StreamingUnsignedTrailer { + chunk_size, + trailer_algorithm, + trailer_value, + } => to_streaming_unsigned_trailer_body( + &self.body, + *chunk_size, + &trailer_algorithm, + &trailer_value, + ), + _ => self.body.clone(), }; let request = request .uri(uri) .method(self.method.clone()) .body(Body::from(body)) .unwrap(); - self.requester.client.request(request).await + + let result = self + .requester + .client + .request(request) + .await + .map_err(|err| format!("hyper client error: {}", err))?; + + let (head, body) = result.into_parts(); + let body = Body::new( + body.collect() + .await + .map_err(|err| format!("hyper client error in body.collect: {}", err))? + .to_bytes(), + ); + Ok(Response::from_parts(head, body)) } } pub enum BodySignature { Unsigned, Classic, - Streaming(usize), + Streaming { + chunk_size: usize, + }, + StreamingUnsignedTrailer { + chunk_size: usize, + trailer_algorithm: String, + trailer_value: String, + }, } fn query_param_to_string(params: &HashMap>) -> String { @@ -303,3 +424,26 @@ fn to_streaming_body( res } + +fn to_streaming_unsigned_trailer_body( + body: &[u8], + chunk_size: usize, + trailer_algorithm: &str, + trailer_value: &str, +) -> Vec { + let mut res = Vec::with_capacity(body.len()); + for chunk in body.chunks(chunk_size) { + let header = format!("{:x}\r\n", chunk.len()); + res.extend_from_slice(header.as_bytes()); + res.extend_from_slice(chunk); + res.extend_from_slice(b"\r\n"); + } + + res.extend_from_slice(b"0\r\n"); + res.extend_from_slice(trailer_algorithm.as_bytes()); + res.extend_from_slice(b":"); + res.extend_from_slice(trailer_value.as_bytes()); + res.extend_from_slice(b"\n\r\n\r\n"); + + res +} diff --git a/src/garage/tests/common/ext/process.rs b/src/garage/tests/common/ext/process.rs index ba533b6c..8e20bf7c 100644 --- a/src/garage/tests/common/ext/process.rs +++ b/src/garage/tests/common/ext/process.rs @@ -14,42 +14,20 @@ impl CommandExt for process::Command { } fn expect_success_status(&mut self, msg: &str) -> process::ExitStatus { - let status = self.status().expect(msg); - status.expect_success(msg); - status + self.expect_success_output(msg).status } fn expect_success_output(&mut self, msg: &str) -> process::Output { let output = self.output().expect(msg); - output.expect_success(msg); + if !output.status.success() { + panic!( + "{}: command {:?} exited with error {:?}\nSTDOUT: {}\nSTDERR: {}", + msg, + self, + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } output } } - -pub trait OutputExt { - fn expect_success(&self, msg: &str); -} - -impl OutputExt for process::Output { - fn expect_success(&self, msg: &str) { - self.status.expect_success(msg) - } -} - -pub trait ExitStatusExt { - fn expect_success(&self, msg: &str); -} - -impl ExitStatusExt for process::ExitStatus { - fn expect_success(&self, msg: &str) { - if !self.success() { - match self.code() { - Some(code) => panic!( - "Command exited with code {code}: {msg}", - code = code, - msg = msg - ), - None => panic!("Command exited with signal: {msg}", msg = msg), - } - } - } -} diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index 8aaf6f5b..2b0a381c 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -13,7 +13,6 @@ static GARAGE_TEST_SECRET: &str = #[derive(Debug, Default, Clone)] pub struct Key { - pub name: Option, pub id: String, pub secret: String, } @@ -42,6 +41,10 @@ impl Instance { .ok() .unwrap_or_else(|| env::temp_dir().join(format!("garage-integ-test-{}", port))); + let db_engine = env::var("GARAGE_TEST_INTEGRATION_DB_ENGINE") + .ok() + .unwrap_or_else(|| "lmdb".into()); + // Clean test runtime directory if path.exists() { fs::remove_dir_all(&path).expect("Could not clean test runtime directory"); @@ -52,13 +55,16 @@ impl Instance { r#" metadata_dir = "{path}/meta" data_dir = "{path}/data" +db_engine = "{db_engine}" -replication_mode = "1" +replication_factor = 1 rpc_bind_addr = "127.0.0.1:{rpc_port}" rpc_public_addr = "127.0.0.1:{rpc_port}" rpc_secret = "{secret}" +allow_punycode = true + [s3_api] s3_region = "{region}" api_bind_addr = "127.0.0.1:{s3_port}" @@ -95,7 +101,10 @@ api_bind_addr = "127.0.0.1:{admin_port}" .arg("server") .stdout(stdout) .stderr(stderr) - .env("RUST_LOG", "garage=info,garage_api=trace") + .env( + "RUST_LOG", + "garage=debug,garage_api_common=trace,garage_api_s3=trace", + ) .spawn() .expect("Could not start garage"); @@ -141,7 +150,7 @@ api_bind_addr = "127.0.0.1:{admin_port}" self.command() .args(["layout", "assign"]) .arg(node_short_id) - .args(["-c", "1", "-z", "unzonned"]) + .args(["-c", "1G", "-z", "unzonned"]) .quiet() .expect_success_status("Could not assign garage node layout"); self.command() @@ -186,9 +195,9 @@ api_bind_addr = "127.0.0.1:{admin_port}" let mut key = Key::default(); let mut cmd = self.command(); - let base = cmd.args(["key", "new"]); + let base = cmd.args(["key", "create"]); let with_name = match maybe_name { - Some(name) => base.args(["--name", name]), + Some(name) => base.args([name]), None => base, }; @@ -208,10 +217,7 @@ api_bind_addr = "127.0.0.1:{admin_port}" assert!(!key.id.is_empty(), "Invalid key: Key ID is empty"); assert!(!key.secret.is_empty(), "Invalid key: Key secret is empty"); - Key { - name: maybe_name.map(String::from), - ..key - } + key } } diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs index eca3e42b..1273bad1 100644 --- a/src/garage/tests/common/mod.rs +++ b/src/garage/tests/common/mod.rs @@ -1,5 +1,8 @@ -use aws_sdk_s3::{Client, Region}; +use aws_sdk_s3::config::Region; +use aws_sdk_s3::Client; use ext::*; +#[cfg(feature = "k2v")] +use k2v_client::K2vClient; #[macro_use] pub mod macros; @@ -19,6 +22,7 @@ pub struct Context { pub key: garage::Key, pub client: Client, pub custom_request: CustomRequester, + #[cfg(feature = "k2v")] pub k2v: K2VContext, } @@ -31,8 +35,9 @@ impl Context { fn new() -> Self { let garage = garage::instance(); let key = garage.key(None); - let client = client::build_client(garage, &key); + let client = client::build_client(&key); let custom_request = CustomRequester::new_s3(garage, &key); + #[cfg(feature = "k2v")] let k2v_request = CustomRequester::new_k2v(garage, &key); Context { @@ -40,6 +45,7 @@ impl Context { client, key, custom_request, + #[cfg(feature = "k2v")] k2v: K2VContext { request: k2v_request, }, @@ -68,6 +74,20 @@ impl Context { bucket_name } + + /// Build a K2vClient for a given bucket + #[cfg(feature = "k2v")] + pub fn k2v_client(&self, bucket: &str) -> K2vClient { + let config = k2v_client::K2vClientConfig { + region: REGION.to_string(), + endpoint: self.garage.k2v_uri().to_string(), + aws_access_key_id: self.key.id.clone(), + aws_secret_access_key: self.key.secret.clone(), + bucket: bucket.to_string(), + user_agent: None, + }; + K2vClient::new(config).expect("Could not create K2V client") + } } pub fn context() -> Context { diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index 595d0ba8..39554d4d 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -6,7 +6,8 @@ use assert_json_diff::assert_json_eq; use base64::prelude::*; use serde_json::json; -use super::json_body; +use crate::json_body; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -77,10 +78,7 @@ async fn test_batch() { .unwrap() .to_string(), ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, values.get(sk).unwrap().as_bytes()); } diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 588836c7..5a347bd9 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -6,7 +6,8 @@ use assert_json_diff::assert_json_eq; use base64::prelude::*; use serde_json::json; -use super::json_body; +use crate::json_body; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -44,6 +45,7 @@ async fn test_items_and_indices() { let content = format!("{}: hello world", sk).into_bytes(); let content2 = format!("{}: hello universe", sk).into_bytes(); let content3 = format!("{}: concurrent value", sk).into_bytes(); + eprintln!("test iteration {}: {}", i, sk); // Put initially, no causality token let res = ctx @@ -82,14 +84,11 @@ async fn test_items_and_indices() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content); // ReadIndex -- now there should be some stuff - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_millis(100)).await; let res = ctx .k2v .request @@ -151,14 +150,11 @@ async fn test_items_and_indices() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content2); // ReadIndex -- now there should be some stuff - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_millis(100)).await; let res = ctx .k2v .request @@ -230,7 +226,7 @@ async fn test_items_and_indices() { ); // ReadIndex -- now there should be some stuff - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_millis(100)).await; let res = ctx .k2v .request @@ -299,7 +295,7 @@ async fn test_items_and_indices() { assert_eq!(res.status(), StatusCode::NO_CONTENT); // ReadIndex -- now there should be some stuff - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_millis(100)).await; let res = ctx .k2v .request @@ -393,10 +389,7 @@ async fn test_item_return_format() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f1: not specified @@ -433,10 +426,7 @@ async fn test_item_return_format() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f3: json diff --git a/src/garage/tests/k2v/mod.rs b/src/garage/tests/k2v/mod.rs index a009460e..241e3dc2 100644 --- a/src/garage/tests/k2v/mod.rs +++ b/src/garage/tests/k2v/mod.rs @@ -3,16 +3,3 @@ pub mod errorcodes; pub mod item; pub mod poll; pub mod simple; - -use hyper::{Body, Response}; - -pub async fn json_body(res: Response) -> serde_json::Value { - let res_body: serde_json::Value = serde_json::from_slice( - &hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec()[..], - ) - .unwrap(); - res_body -} diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index dd44aed9..7c06cea9 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -1,14 +1,16 @@ use base64::prelude::*; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; use std::time::Duration; use assert_json_diff::assert_json_eq; use serde_json::json; -use super::json_body; use crate::common; +use crate::json_body; #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_item() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-item"); @@ -47,11 +49,8 @@ async fn test_poll_item() { .unwrap() .to_string(); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Initial value"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Initial value"[..]); // Start poll operation let poll = { @@ -95,14 +94,12 @@ async fn test_poll_item() { assert_eq!(poll_res.status(), StatusCode::OK); - let poll_res_body = hyper::body::to_bytes(poll_res.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(poll_res_body, b"New value"); + let poll_res_body = poll_res.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(poll_res_body, b"New value"[..]); } #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_range() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-range"); diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs index 465fc24d..1017330d 100644 --- a/src/garage/tests/k2v/simple.rs +++ b/src/garage/tests/k2v/simple.rs @@ -1,5 +1,6 @@ use crate::common; +use http_body_util::BodyExt; use hyper::{Method, StatusCode}; #[tokio::test] @@ -32,9 +33,6 @@ async fn test_simple() { .unwrap(); assert_eq!(res2.status(), StatusCode::OK); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Hello, world!"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Hello, world!"[..]); } diff --git a/src/garage/tests/k2v_client/mod.rs b/src/garage/tests/k2v_client/mod.rs new file mode 100644 index 00000000..b252f36b --- /dev/null +++ b/src/garage/tests/k2v_client/mod.rs @@ -0,0 +1 @@ +pub mod simple; diff --git a/src/garage/tests/k2v_client/simple.rs b/src/garage/tests/k2v_client/simple.rs new file mode 100644 index 00000000..1a3118ef --- /dev/null +++ b/src/garage/tests/k2v_client/simple.rs @@ -0,0 +1,60 @@ +use std::time::Duration; + +use k2v_client::*; + +use crate::common; + +#[tokio::test] +async fn test_simple() { + let ctx = common::context(); + let bucket = ctx.create_bucket("test-k2v-client-simple"); + let k2v_client = ctx.k2v_client(&bucket); + + k2v_client + .insert_item("root", "test1", b"Hello, world!".to_vec(), None) + .await + .unwrap(); + + let res = k2v_client.read_item("root", "test1").await.unwrap(); + + assert_eq!(res.value.len(), 1); + assert_eq!(res.value[0], K2vValue::Value(b"Hello, world!".to_vec())); +} + +#[tokio::test] +async fn test_special_chars() { + let ctx = common::context(); + let bucket = ctx.create_bucket("test-k2v-client-simple-special-chars"); + let k2v_client = ctx.k2v_client(&bucket); + + let (pk, sk) = ("root@plépp", "≤≤««"); + k2v_client + .insert_item(pk, sk, b"Hello, world!".to_vec(), None) + .await + .unwrap(); + + let res = k2v_client.read_item(pk, sk).await.unwrap(); + assert_eq!(res.value.len(), 1); + assert_eq!(res.value[0], K2vValue::Value(b"Hello, world!".to_vec())); + + // sleep a bit before read_index + tokio::time::sleep(Duration::from_secs(1)).await; + let res = k2v_client.read_index(Default::default()).await.unwrap(); + assert_eq!(res.items.len(), 1); + assert_eq!(res.items.keys().next().unwrap(), pk); + + let res = k2v_client + .read_batch(&[BatchReadOp { + partition_key: pk, + filter: Default::default(), + single_item: false, + conflicts_only: false, + tombstones: false, + }]) + .await + .unwrap(); + assert_eq!(res.len(), 1); + let res = &res[0]; + assert_eq!(res.items.len(), 1); + assert_eq!(res.items.keys().next().unwrap(), sk); +} diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 87be1327..ef370db3 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -8,3 +8,18 @@ mod s3; #[cfg(feature = "k2v")] mod k2v; +#[cfg(feature = "k2v")] +mod k2v_client; + +use http_body_util::BodyExt; +use hyper::{body::Body, Response}; + +pub async fn json_body(res: Response) -> serde_json::Value +where + B: Body, + ::Error: std::fmt::Debug, +{ + let body = res.into_body().collect().await.unwrap().to_bytes(); + let res_body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + res_body +} diff --git a/src/garage/tests/s3/list.rs b/src/garage/tests/s3/list.rs index bb03f250..1b0c006d 100644 --- a/src/garage/tests/s3/list.rs +++ b/src/garage/tests/s3/list.rs @@ -613,3 +613,63 @@ async fn test_listmultipart() { assert!(r.common_prefixes.is_none()); } } + +#[tokio::test] +async fn test_multichar_delimiter() { + // Test case from dpape from issue #692 with reference results from Amazon + + let ctx = common::context(); + let bucket = ctx.create_bucket("multichardelim"); + + for k in [ + "a/", "a/b/", "a/b/c/", "a/b/c/d", "a/c/", "a/c/b/", "a/c/b/e", + ] { + ctx.client + .put_object() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + // With delimiter / + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert!(r.contents.is_none()); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 1); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/"); + } + + // With delimiter b/ + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("b/") + .send() + .await + .unwrap(); + + let contents = r.contents.unwrap(); + assert_eq!(contents.len(), 2); + assert_eq!(contents[0].key.as_deref().unwrap(), "a/"); + assert_eq!(contents[1].key.as_deref().unwrap(), "a/c/"); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 2); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/b/"); + assert_eq!(common_prefixes[1].prefix.as_deref().unwrap(), "a/c/b/"); + } +} diff --git a/src/garage/tests/s3/mod.rs b/src/garage/tests/s3/mod.rs index 623eb665..e75b1397 100644 --- a/src/garage/tests/s3/mod.rs +++ b/src/garage/tests/s3/mod.rs @@ -1,6 +1,8 @@ mod list; mod multipart; mod objects; +mod presigned; mod simple; +mod ssec; mod streaming_signature; mod website; diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs index 895a2993..cc424f59 100644 --- a/src/garage/tests/s3/multipart.rs +++ b/src/garage/tests/s3/multipart.rs @@ -1,10 +1,342 @@ use crate::common; -use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; -use aws_sdk_s3::types::ByteStream; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart}; +use base64::prelude::*; const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024; +#[tokio::test] +async fn test_multipart_upload() { + let ctx = common::context(); + let bucket = ctx.create_bucket("testmpu"); + + let u1 = vec![0x11; SZ_5MB]; + let u2 = vec![0x22; SZ_5MB]; + let u3 = vec![0x33; SZ_5MB]; + let u4 = vec![0x44; SZ_5MB]; + let u5 = vec![0x55; SZ_5MB]; + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + assert!(up.upload_id.is_some()); + + let uid = up.upload_id.as_ref().unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(3) + .body(ByteStream::from(u3.clone())) + .send() + .await + .unwrap(); + + let _p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u1)) + .send() + .await + .unwrap(); + + let _p4 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(4) + .body(ByteStream::from(u4)) + .send() + .await + .unwrap(); + + let p1bis = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u2.clone())) + .send() + .await + .unwrap(); + + let p6 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(6) + .body(ByteStream::from(u5.clone())) + .send() + .await + .unwrap(); + + { + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + assert_eq!(r.parts.unwrap().len(), 4); + } + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1bis.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(6) + .e_tag(p6.e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // The multipart upload must not appear anymore + assert!(ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .is_err()); + + { + // The object must appear as a regular object + let r = ctx + .client + .head_object() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 3) as i64); + } + + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, &[&u2[..], &u3[..], &u5[..]].concat()); + } + + { + for (part_number, data) in [(1, &u2), (2, &u3), (3, &u5)] { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key("a") + .part_number(part_number) + .send() + .await + .unwrap(); + + eprintln!("get_object with part_number = {}", part_number); + assert_eq!(o.content_length.unwrap(), SZ_5MB as i64); + assert_bytes_eq!(o.body, data); + } + } +} + +#[tokio::test] +async fn test_multipart_with_checksum() { + let ctx = common::context(); + let bucket = ctx.create_bucket("testmpu-cksum"); + + let u1 = vec![0x11; SZ_5MB]; + let u2 = vec![0x22; SZ_5MB]; + let u3 = vec![0x33; SZ_5MB]; + + let ck1 = calculate_sha1(&u1); + let ck2 = calculate_sha1(&u2); + let ck3 = calculate_sha1(&u3); + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .checksum_algorithm(ChecksumAlgorithm::Sha1) + .key("a") + .send() + .await + .unwrap(); + assert!(up.upload_id.is_some()); + + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .checksum_sha1(&ck1) + .body(ByteStream::from(u1.clone())) + .send() + .await + .unwrap(); + + // wrong checksum value should return an error + let err1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .checksum_sha1(&ck1) + .body(ByteStream::from(u2.clone())) + .send() + .await; + assert!(err1.is_err()); + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .checksum_sha1(&ck2) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(3) + .checksum_sha1(&ck3) + .body(ByteStream::from(u3.clone())) + .send() + .await + .unwrap(); + + { + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + let parts = r.parts.unwrap(); + assert_eq!(parts.len(), 3); + assert!(parts[0].checksum_crc32.is_none()); + assert!(parts[0].checksum_crc32_c.is_none()); + assert!(parts[0].checksum_sha256.is_none()); + assert_eq!(parts[0].checksum_sha1.as_deref().unwrap(), ck1); + assert_eq!(parts[1].checksum_sha1.as_deref().unwrap(), ck2); + assert_eq!(parts[2].checksum_sha1.as_deref().unwrap(), ck3); + } + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .checksum_sha1(&ck1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .checksum_sha1(&ck2) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .checksum_sha1(&ck3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .build(); + + let expected_checksum = calculate_sha1( + &vec![ + BASE64_STANDARD.decode(&ck1).unwrap(), + BASE64_STANDARD.decode(&ck2).unwrap(), + BASE64_STANDARD.decode(&ck3).unwrap(), + ] + .concat(), + ); + + let res = ctx + .client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .checksum_sha1(expected_checksum.clone()) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + assert_eq!(res.checksum_sha1, Some(expected_checksum)); +} + #[tokio::test] async fn test_uploadlistpart() { let ctx = common::context(); @@ -65,13 +397,14 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 1); - let fp = ps.iter().find(|x| x.part_number == 2).unwrap(); + assert_eq!(ps[0].part_number.unwrap(), 2); + let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); } let p2 = ctx @@ -100,13 +433,24 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 2); - let fp = ps.iter().find(|x| x.part_number == 1).unwrap(); + + assert_eq!(ps[0].part_number.unwrap(), 1); + let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3c484266f9315485694556e6c693bfa2\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); + + assert_eq!(ps[1].part_number.unwrap(), 2); + let sp = &ps[1]; + assert!(sp.last_modified.is_some()); + assert_eq!( + sp.e_tag.as_ref().unwrap(), + "\"3366bb9dcf710d6801b5926467d02e19\"" + ); + assert_eq!(sp.size.unwrap(), SZ_5MB as i64); } { @@ -123,12 +467,19 @@ async fn test_uploadlistpart() { .unwrap(); assert!(r.part_number_marker.is_none()); - assert!(r.next_part_number_marker.is_some()); - assert_eq!(r.max_parts, 1_i32); - assert!(r.is_truncated); + assert_eq!(r.next_part_number_marker.as_deref(), Some("1")); + assert_eq!(r.max_parts.unwrap(), 1_i32); + assert!(r.is_truncated.unwrap()); assert_eq!(r.key.unwrap(), "a"); assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str()); - assert_eq!(r.parts.unwrap().len(), 1); + let parts = r.parts.unwrap(); + assert_eq!(parts.len(), 1); + let fp = &parts[0]; + assert_eq!(fp.part_number.unwrap(), 1); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3c484266f9315485694556e6c693bfa2\"" + ); let r2 = ctx .client @@ -146,11 +497,19 @@ async fn test_uploadlistpart() { r2.part_number_marker.as_ref().unwrap(), r.next_part_number_marker.as_ref().unwrap() ); - assert_eq!(r2.max_parts, 1_i32); - assert!(r2.is_truncated); + assert_eq!(r2.max_parts.unwrap(), 1_i32); assert_eq!(r2.key.unwrap(), "a"); assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str()); - assert_eq!(r2.parts.unwrap().len(), 1); + let parts = r2.parts.unwrap(); + assert_eq!(parts.len(), 1); + let fp = &parts[0]; + assert_eq!(fp.part_number.unwrap(), 2); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3366bb9dcf710d6801b5926467d02e19\"" + ); + //assert!(r2.is_truncated); // WHY? (this was the test before) + assert!(!r2.is_truncated.unwrap()); } let cmp = CompletedMultipartUpload::builder() @@ -200,7 +559,7 @@ async fn test_uploadlistpart() { .await .unwrap(); - assert_eq!(r.content_length, (SZ_5MB * 2) as i64); + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 2) as i64); } } @@ -413,3 +772,11 @@ async fn test_uploadpartcopy() { assert_eq!(real_obj.len(), exp_obj.len()); assert_eq!(real_obj, exp_obj); } + +fn calculate_sha1(bytes: &[u8]) -> String { + use sha1::{Digest, Sha1}; + + let mut hasher = Sha1::new(); + hasher.update(bytes); + BASE64_STANDARD.encode(&hasher.finalize()[..]) +} diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index 65f9e867..53e8231d 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -1,6 +1,7 @@ use crate::common; -use aws_sdk_s3::model::{Delete, ObjectIdentifier}; -use aws_sdk_s3::types::ByteStream; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::primitives::{ByteStream, DateTime}; +use aws_sdk_s3::types::{Delete, ObjectIdentifier}; const STD_KEY: &str = "hello world"; const CTRL_KEY: &str = "\x00\x01\x02\x00"; @@ -50,9 +51,9 @@ async fn test_putobject() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -86,9 +87,9 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b"hi"); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 2); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 2); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -119,9 +120,156 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b""); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + } +} + +#[tokio::test] +async fn test_precondition() { + let ctx = common::context(); + let bucket = ctx.create_bucket("precondition"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let etag2 = "\"ae4984b984cd984fe98d4efa954dce98\""; + let data = ByteStream::from_static(BODY); + + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let last_modified; + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_match(etag) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + last_modified = o.last_modified.unwrap(); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_match(etag2) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) + ); + } + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_none_match(etag2) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_none_match(etag) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + } + let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0); + let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64()); + let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0); + { + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(newer_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(same_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(older_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + } + { + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(older_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) + ); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(same_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(newer_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); } } @@ -187,6 +335,148 @@ async fn test_getobject() { } } +#[tokio::test] +async fn test_metadata() { + use aws_sdk_s3::primitives::{DateTime, DateTimeFormat}; + + let ctx = common::context(); + let bucket = ctx.create_bucket("testmetadata"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let exp = DateTime::from_secs(10000000000); + let exp2 = DateTime::from_secs(10000500000); + + { + // Note. The AWS client SDK adds a Content-Type header + // with value application/octet-stream if it is not given, + // so here we force it to a known different value. + let data = ByteStream::from_static(BODY); + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .content_type("application/test") + .send() + .await + .unwrap(); + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let o = ctx + .client + .head_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/test"); + assert_eq!(o.cache_control, None); + assert_eq!(o.content_disposition, None); + assert_eq!(o.content_encoding, None); + assert_eq!(o.content_language, None); + assert_eq!(o.expires_string, None); + assert_eq!(o.metadata.unwrap_or_default().len(), 0); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .response_content_type("application/x-dummy-test") + .response_cache_control("ccdummy") + .response_content_disposition("cddummy") + .response_content_encoding("cedummy") + .response_content_language("cldummy") + .response_expires(exp) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test"); + assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); + assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); + } + + { + let data = ByteStream::from_static(BODY); + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .content_type("application/test") + .cache_control("cctest") + .content_disposition("cdtest") + .content_encoding("cetest") + .content_language("cltest") + .expires(exp2) + .metadata("testmeta", "hello people") + .metadata("nice-unicode-meta", "宅配便") + .send() + .await + .unwrap(); + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + let o = ctx + .client + .head_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/test"); + assert_eq!(o.cache_control.unwrap().as_str(), "cctest"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cdtest"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cetest"); + assert_eq!(o.content_language.unwrap().as_str(), "cltest"); + assert_eq!( + o.expires_string.unwrap(), + exp2.fmt(DateTimeFormat::HttpDate).unwrap() + ); + let mut meta = o.metadata.unwrap(); + assert_eq!(meta.remove("testmeta").unwrap(), "hello people"); + assert_eq!(meta.remove("nice-unicode-meta").unwrap(), "宅配便"); + assert_eq!(meta.len(), 0); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .response_content_type("application/x-dummy-test") + .response_cache_control("ccdummy") + .response_content_disposition("cddummy") + .response_content_encoding("cedummy") + .response_content_language("cldummy") + .response_expires(exp) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.unwrap().as_str(), etag); + assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test"); + assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); + assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); + } +} + #[tokio::test] async fn test_deleteobject() { let ctx = common::context(); @@ -205,7 +495,7 @@ async fn test_deleteobject() { .await .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -223,7 +513,7 @@ async fn test_deleteobject() { .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -247,7 +537,7 @@ async fn test_deleteobject() { .client .delete_objects() .bucket(&bucket) - .delete(to_del.build()) + .delete(to_del.build().unwrap()) .send() .await .unwrap(); diff --git a/src/garage/tests/s3/presigned.rs b/src/garage/tests/s3/presigned.rs new file mode 100644 index 00000000..15270361 --- /dev/null +++ b/src/garage/tests/s3/presigned.rs @@ -0,0 +1,72 @@ +use std::time::{Duration, SystemTime}; + +use crate::common; +use aws_sdk_s3::presigning::PresigningConfig; +use bytes::Bytes; +use http_body_util::{BodyExt, Full}; +use hyper::Request; + +const STD_KEY: &str = "hello world"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_presigned_url() { + let ctx = common::context(); + let bucket = ctx.create_bucket("presigned"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let body = Bytes::from(BODY.to_vec()); + + let psc = PresigningConfig::builder() + .start_time(SystemTime::now() - Duration::from_secs(60)) + .expires_in(Duration::from_secs(3600)) + .build() + .unwrap(); + + { + // PutObject + let req = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc.clone()) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("PUT") + .uri(req.uri()) + .body(Full::new(body.clone())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + } + + { + // GetObject + let req = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("GET") + .uri(req.uri()) + .body(Full::new(Bytes::new())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + + let body2 = BodyExt::collect(res.into_body()).await.unwrap().to_bytes(); + assert_eq!(body, body2); + } +} diff --git a/src/garage/tests/s3/simple.rs b/src/garage/tests/s3/simple.rs index f54ae9ac..41ec44c6 100644 --- a/src/garage/tests/s3/simple.rs +++ b/src/garage/tests/s3/simple.rs @@ -2,7 +2,7 @@ use crate::common; #[tokio::test] async fn test_simple() { - use aws_sdk_s3::types::ByteStream; + use aws_sdk_s3::primitives::ByteStream; let ctx = common::context(); let bucket = ctx.create_bucket("test-simple"); diff --git a/src/garage/tests/s3/ssec.rs b/src/garage/tests/s3/ssec.rs new file mode 100644 index 00000000..d8f11950 --- /dev/null +++ b/src/garage/tests/s3/ssec.rs @@ -0,0 +1,455 @@ +use crate::common::{self, Context}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + +const SSEC_KEY: &str = "u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y="; +const SSEC_KEY_MD5: &str = "jMGbs3GyZkYjJUP6q5jA7g=="; +const SSEC_KEY2: &str = "XkYVk4Z3vVDO2yJaUqCAEZX6lL10voMxtV06d8my/eU="; +const SSEC_KEY2_MD5: &str = "kedo2ab8J1MCjHwJuLTJHw=="; + +const SZ_2MB: usize = 2 * 1024 * 1024; + +#[tokio::test] +async fn test_ssec_object() { + let ctx = common::context(); + let bucket = ctx.create_bucket("sse-c"); + + let bytes1 = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec(); + let bytes2 = (0..400000) + .map(|x| ((x * 3792) % 256) as u8) + .collect::>(); + + for data in vec![bytes1, bytes2] { + let stream = ByteStream::new(data.clone().into()); + + // Write encrypted object + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key("testobj") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .body(stream) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + + test_read_encrypted( + &ctx, + &bucket, + "testobj", + &data, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + + // Test copy from encrypted to non-encrypted + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-dec") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, None); + assert_eq!(r.sse_customer_key_md5, None); + + // Test read decrypted file + let r = ctx + .client + .get_object() + .bucket(&bucket) + .key("test-copy-enc-dec") + .send() + .await + .unwrap(); + assert_bytes_eq!(r.body, &data); + assert_eq!(r.sse_customer_algorithm, None); + assert_eq!(r.sse_customer_key_md5, None); + + // Test copy from non-encrypted to encrypted + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-dec-enc") + .copy_source(format!("{}/test-copy-enc-dec", bucket)) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into())); + + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-dec-enc", + &data, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + + // Test copy from encrypted to encrypted with different keys + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-enc") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into())); + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-enc", + &data, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + + // Test copy from encrypted to encrypted with the same key + let r = ctx + .client + .copy_object() + .bucket(&bucket) + .key("test-copy-enc-enc-same") + .copy_source(format!("{}/{}", bucket, "testobj")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert_eq!(r.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + test_read_encrypted( + &ctx, + &bucket, + "test-copy-enc-enc-same", + &data, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + } +} + +#[tokio::test] +async fn test_multipart_upload() { + let ctx = common::context(); + let bucket = ctx.create_bucket("test-ssec-mpu"); + + let u1 = vec![0x11; SZ_2MB]; + let u2 = vec![0x22; SZ_2MB]; + let u3 = vec![0x33; SZ_2MB]; + let all = [&u1[..], &u2[..], &u3[..]].concat(); + + // Test simple encrypted mpu + { + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("a") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .send() + .await + .unwrap(); + assert!(up.upload_id.is_some()); + assert_eq!(up.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(up.sse_customer_key_md5, Some(SSEC_KEY_MD5.into())); + + let uid = up.upload_id.as_ref().unwrap(); + + let mut etags = vec![]; + for (i, part) in vec![&u1, &u2, &u3].into_iter().enumerate() { + let pu = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number((i + 1) as i32) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY) + .sse_customer_key_md5(SSEC_KEY_MD5) + .body(ByteStream::from(part.to_vec())) + .send() + .await + .unwrap(); + etags.push(pu.e_tag.unwrap()); + } + + let mut cmp = CompletedMultipartUpload::builder(); + for (i, etag) in etags.into_iter().enumerate() { + cmp = cmp.parts( + CompletedPart::builder() + .part_number((i + 1) as i32) + .e_tag(etag) + .build(), + ); + } + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .multipart_upload(cmp.build()) + .send() + .await + .unwrap(); + + test_read_encrypted( + &ctx, + &bucket, + "a", + &all, + SSEC_KEY, + SSEC_KEY_MD5, + SSEC_KEY2, + SSEC_KEY2_MD5, + ) + .await; + } + + // Test upload part copy from first object + { + // (setup) Upload a single part object + ctx.client + .put_object() + .bucket(&bucket) + .key("b") + .body(ByteStream::from(u1.clone())) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("target") + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(1) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .body(ByteStream::from(u3.clone())) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(2) + .copy_source(format!("{}/a", bucket)) + .copy_source_range("bytes=500-550000") + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY) + .copy_source_sse_customer_key_md5(SSEC_KEY_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(3) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .body(ByteStream::from(u2.clone())) + .send() + .await + .unwrap(); + + let p4 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(4) + .copy_source(format!("{}/b", bucket)) + .copy_source_range("bytes=1500-20500") + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(SSEC_KEY2) + .copy_source_sse_customer_key_md5(SSEC_KEY2_MD5) + .sse_customer_algorithm("AES256") + .sse_customer_key(SSEC_KEY2) + .sse_customer_key_md5(SSEC_KEY2_MD5) + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(4) + .e_tag(p4.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // (check) Get object + let expected = [&u3[..], &all[500..550001], &u2[..], &u1[1500..20501]].concat(); + test_read_encrypted( + &ctx, + &bucket, + "target", + &expected, + SSEC_KEY2, + SSEC_KEY2_MD5, + SSEC_KEY, + SSEC_KEY_MD5, + ) + .await; + } +} + +async fn test_read_encrypted( + ctx: &Context, + bucket: &str, + obj_key: &str, + expected_data: &[u8], + enc_key: &str, + enc_key_md5: &str, + wrong_enc_key: &str, + wrong_enc_key_md5: &str, +) { + // Test read encrypted without key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .send() + .await; + assert!( + o.is_err(), + "encrypted file could be read without encryption key" + ); + + // Test read encrypted with wrong key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .sse_customer_key(wrong_enc_key) + .sse_customer_key_md5(wrong_enc_key_md5) + .send() + .await; + assert!( + o.is_err(), + "encrypted file could be read with incorrect encryption key" + ); + + // Test read encrypted with correct key + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(obj_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(enc_key) + .sse_customer_key_md5(enc_key_md5) + .send() + .await + .unwrap(); + assert_bytes_eq!(o.body, expected_data); + assert_eq!(o.sse_customer_algorithm, Some("AES256".into())); + assert_eq!(o.sse_customer_key_md5, Some(enc_key_md5.to_string())); +} diff --git a/src/garage/tests/s3/streaming_signature.rs b/src/garage/tests/s3/streaming_signature.rs index b7a1acae..a86feefc 100644 --- a/src/garage/tests/s3/streaming_signature.rs +++ b/src/garage/tests/s3/streaming_signature.rs @@ -1,5 +1,8 @@ use std::collections::HashMap; +use base64::prelude::*; +use crc32fast::Hasher as Crc32; + use crate::common; use crate::common::ext::CommandExt; use common::custom_requester::BodySignature; @@ -21,18 +24,19 @@ async fn test_putobject_streaming() { let content_type = "text/csv"; let mut headers = HashMap::new(); headers.insert("content-type".to_owned(), content_type.to_owned()); - let _ = ctx + let res = ctx .custom_request .builder(bucket.clone()) .method(Method::PUT) .path(STD_KEY.to_owned()) - .unsigned_headers(headers) + .signed_headers(headers) .vhost_style(true) .body(vec![]) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); // assert_eq!(r.e_tag.unwrap().as_str(), etag); // We return a version ID here @@ -57,15 +61,22 @@ async fn test_putobject_streaming() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { let etag = "\"46cf18a9b447991b450cad3facf5937e\""; - let _ = ctx + let mut crc32 = Crc32::new(); + crc32.update(&BODY[..]); + let crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(crc32.finalize())[..]); + + let mut headers = HashMap::new(); + headers.insert("x-amz-checksum-crc32".to_owned(), crc32.clone()); + + let res = ctx .custom_request .builder(bucket.clone()) .method(Method::PUT) @@ -73,11 +84,13 @@ async fn test_putobject_streaming() { //fail .path("abc".to_owned()) .vhost_style(true) + .signed_headers(headers) .body(BODY.to_vec()) - .body_signature(BodySignature::Streaming(16)) + .body_signature(BodySignature::Streaming { chunk_size: 16 }) .send() .await .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); // assert_eq!(r.e_tag.unwrap().as_str(), etag); // assert!(r.version_id.is_some()); @@ -88,6 +101,7 @@ async fn test_putobject_streaming() { .bucket(&bucket) //.key(CTRL_KEY) .key("abc") + .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) .send() .await .unwrap(); @@ -95,9 +109,145 @@ async fn test_putobject_streaming() { assert_bytes_eq!(o.body, BODY); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 62); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 62); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + assert_eq!(o.checksum_crc32.unwrap(), crc32); + } +} + +#[tokio::test] +async fn test_putobject_streaming_unsigned_trailer() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putobject-streaming-unsigned-trailer"); + + { + // Send an empty object (can serve as a directory marker) + // with a content type + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let mut headers = HashMap::new(); + headers.insert("content-type".to_owned(), content_type.to_owned()); + + let empty_crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(Crc32::new().finalize())[..]); + + let res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + .path(STD_KEY.to_owned()) + .signed_headers(headers) + .vhost_style(true) + .body(vec![]) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 10, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: empty_crc32, + }) + .send() + .await + .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // We return a version ID here + // We should check if Amazon is returning one when versioning is not enabled + // assert!(r.version_id.is_some()); + + //let _version = r.version_id.unwrap(); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + // We do not return version ID + // We should check if Amazon is returning one when versioning is not enabled + // assert_eq!(o.version_id.unwrap(), _version); + assert_eq!(o.content_type.unwrap(), content_type); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + } + + { + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + + let mut crc32 = Crc32::new(); + crc32.update(&BODY[..]); + let crc32 = BASE64_STANDARD.encode(&u32::to_be_bytes(crc32.finalize())[..]); + + // try sending with wrong crc32, check that it fails + let err_res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 16, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: "2Yp9Yw==".into(), + }) + .send() + .await + .unwrap(); + assert!( + err_res.status().is_client_error(), + "got response: {:?}", + err_res + ); + + let res = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::StreamingUnsignedTrailer { + chunk_size: 16, + trailer_algorithm: "x-amz-checksum-crc32".into(), + trailer_value: crc32.clone(), + }) + .send() + .await + .unwrap(); + assert!(res.status().is_success(), "got response: {:?}", res); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + //.key(CTRL_KEY) + .key("abc") + .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, BODY); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length.unwrap(), 62); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); + assert_eq!(o.checksum_crc32.unwrap(), crc32); } } @@ -119,7 +269,7 @@ async fn test_create_bucket_streaming() { .custom_request .builder(bucket.to_owned()) .method(Method::PUT) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); @@ -174,7 +324,7 @@ async fn test_put_website_streaming() { .method(Method::PUT) .query_params(query) .body(website_config.as_bytes().to_vec()) - .body_signature(BodySignature::Streaming(10)) + .body_signature(BodySignature::Streaming { chunk_size: 10 }) .send() .await .unwrap(); @@ -187,7 +337,7 @@ async fn test_put_website_streaming() { .await .unwrap(); - assert_eq!(o.index_document.unwrap().suffix.unwrap(), "home.html"); - assert_eq!(o.error_document.unwrap().key.unwrap(), "err/error.html"); + assert_eq!(o.index_document.unwrap().suffix, "home.html"); + assert_eq!(o.error_document.unwrap().key, "err/error.html"); } } diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index f61838e4..bbac3de5 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -1,22 +1,26 @@ use crate::common; use crate::common::ext::*; -use crate::k2v::json_body; +use crate::json_body; use assert_json_diff::assert_json_eq; use aws_sdk_s3::{ - model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, - types::ByteStream, + primitives::ByteStream, + types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, }; use http::{Request, StatusCode}; -use hyper::{ - body::{to_bytes, Body}, - Client, -}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::body::Bytes; +use hyper::header::LOCATION; +use hyper_util::client::legacy::Client; +use hyper_util::rt::TokioExecutor; use serde_json::json; const BODY: &[u8; 16] = b"

bonjour

"; const BODY_ERR: &[u8; 6] = b"erreur"; +pub type Body = FullBody; + #[tokio::test] async fn test_website() { const BCKT_NAME: &str = "my-website"; @@ -34,14 +38,14 @@ async fn test_website() { .await .unwrap(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -49,7 +53,7 @@ async fn test_website() { assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BodyExt::collect(resp.into_body()).await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -58,10 +62,9 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -72,7 +75,7 @@ async fn test_website() { res_body, json!({ "code": "InvalidRequest", - "message": "Bad request: Bucket 'my-website' is not authorized for website hosting", + "message": "Bad request: Domain 'my-website' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -87,28 +90,33 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); - let admin_req = || { - Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() - )) - .body(Body::empty()) - .unwrap() - }; + for bname in [ + BCKT_NAME.to_string(), + format!("{BCKT_NAME}.web.garage"), + format!("{BCKT_NAME}.s3.garage"), + ] { + let admin_req = || { + Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{0}/check?domain={1}", + ctx.garage.admin_port, bname + )) + .body(Body::new(Bytes::new())) + .unwrap() + }; - let mut admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::OK); - assert_eq!( - to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), - format!("Bucket '{BCKT_NAME}' is authorized for website hosting").as_bytes() - ); + let admin_resp = client.request(admin_req()).await.unwrap(); + assert_eq!(admin_resp.status(), StatusCode::OK); + assert_eq!( + admin_resp.into_body().collect().await.unwrap().to_bytes(), + format!("Domain '{bname}' is managed by Garage").as_bytes() + ); + } ctx.garage .command() @@ -119,7 +127,7 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -128,10 +136,9 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -142,7 +149,7 @@ async fn test_website() { res_body, json!({ "code": "InvalidRequest", - "message": "Bad request: Bucket 'my-website' is not authorized for website hosting", + "message": "Bad request: Domain 'my-website' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -176,8 +183,18 @@ async fn test_website_s3_api() { .unwrap(); let conf = WebsiteConfiguration::builder() - .index_document(IndexDocument::builder().suffix("home.html").build()) - .error_document(ErrorDocument::builder().key("err/error.html").build()) + .index_document( + IndexDocument::builder() + .suffix("home.html") + .build() + .unwrap(), + ) + .error_document( + ErrorDocument::builder() + .key("err/error.html") + .build() + .unwrap(), + ) .build(); ctx.client @@ -196,9 +213,11 @@ async fn test_website_s3_api() { .allowed_methods("GET") .allowed_methods("PUT") .allowed_origins("*") - .build(), + .build() + .unwrap(), ) - .build(); + .build() + .unwrap(); ctx.client .put_bucket_cors() @@ -217,24 +236,21 @@ async fn test_website_s3_api() { .await .unwrap(); - let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap(); + let main_rule = cors_res.cors_rules().iter().next().unwrap(); assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule"); assert_eq!( main_rule.allowed_headers.as_ref().unwrap(), &vec!["*".to_string()] ); + assert_eq!(&main_rule.allowed_origins, &vec!["*".to_string()]); assert_eq!( - main_rule.allowed_origins.as_ref().unwrap(), - &vec!["*".to_string()] - ); - assert_eq!( - main_rule.allowed_methods.as_ref().unwrap(), + &main_rule.allowed_methods, &vec!["GET".to_string(), "PUT".to_string()] ); } - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); // Test direct requests with CORS { @@ -243,10 +259,10 @@ async fn test_website_s3_api() { .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -254,7 +270,7 @@ async fn test_website_s3_api() { "*" ); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -268,18 +284,45 @@ async fn test_website_s3_api() { ctx.garage.web_port )) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY_ERR.as_ref() ); } + // Test x-amz-website-redirect-location + { + ctx.client + .put_object() + .bucket(&bucket) + .key("test-redirect.html") + .website_redirect_location("https://perdu.com") + .send() + .await + .unwrap(); + + let req = Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/test-redirect.html", + ctx.garage.web_port + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .body(Body::new(Bytes::new())) + .unwrap(); + + let resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), StatusCode::MOVED_PERMANENTLY); + assert_eq!(resp.headers().get(LOCATION).unwrap(), "https://perdu.com"); + } + // Test CORS with an allowed preflight request { let req = Request::builder() @@ -288,10 +331,10 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -299,7 +342,7 @@ async fn test_website_s3_api() { "*" ); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -312,14 +355,14 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "DELETE") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -353,14 +396,14 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -379,34 +422,29 @@ async fn test_website_s3_api() { .method("GET") .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY_ERR.as_ref() - ); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); + let resp_bytes = resp.into_body().collect().await.unwrap().to_bytes(); + assert_ne!(resp_bytes, BODY_ERR.as_ref()); + assert_ne!(resp_bytes, BODY.as_ref()); } } #[tokio::test] -async fn test_website_check_website_enabled() { +async fn test_website_check_domain() { let ctx = common::context(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let admin_req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/check", ctx.garage.admin_port)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -430,18 +468,18 @@ async fn test_website_check_website_enabled() { "http://127.0.0.1:{}/check?domain=", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: ", + "code": "InvalidRequest", + "message": "Bad request: Domain '' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -454,18 +492,18 @@ async fn test_website_check_website_enabled() { "http://127.0.0.1:{}/check?domain=foobar", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: foobar", + "code": "InvalidRequest", + "message": "Bad request: Domain 'foobar' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -478,20 +516,135 @@ async fn test_website_check_website_enabled() { "http://127.0.0.1:{}/check?domain=%E2%98%B9", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: ☹", + "code": "InvalidRequest", + "message": "Bad request: Domain '☹' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) ); } + +#[tokio::test] +async fn test_website_puny() { + const BCKT_NAME: &str = "xn--pda.eu"; + let ctx = common::context(); + let bucket = ctx.create_bucket(BCKT_NAME); + + let data = ByteStream::from_static(BODY); + + ctx.client + .put_object() + .bucket(&bucket) + .key("index.html") + .body(data) + .send() + .await + .unwrap(); + + let client = Client::builder(TokioExecutor::new()).build_http(); + + let req = |suffix| { + Request::builder() + .method("GET") + .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) + .header("Host", format!("{}{}", BCKT_NAME, suffix)) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + let mut resp = client.request(req("")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.into_body().collect().await.unwrap().to_bytes(), + BODY.as_ref() + ); + + resp = client.request(req(".web.garage")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.into_body().collect().await.unwrap().to_bytes(), + BODY.as_ref() + ); + + for bname in [ + BCKT_NAME.to_string(), + format!("{BCKT_NAME}.web.garage"), + format!("{BCKT_NAME}.s3.garage"), + ] { + let admin_req = || { + Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{0}/check?domain={1}", + ctx.garage.admin_port, bname + )) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + let admin_resp = client.request(admin_req()).await.unwrap(); + assert_eq!(admin_resp.status(), StatusCode::OK); + assert_eq!( + admin_resp.into_body().collect().await.unwrap().to_bytes(), + format!("Domain '{bname}' is managed by Garage").as_bytes() + ); + } +} + +#[tokio::test] +async fn test_website_object_not_found() { + const BCKT_NAME: &str = "not-found"; + let ctx = common::context(); + let _bucket = ctx.create_bucket(BCKT_NAME); + + let client = Client::builder(TokioExecutor::new()).build_http(); + + let req = |suffix| { + Request::builder() + .method("GET") + .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) + .header("Host", format!("{}{}", BCKT_NAME, suffix)) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + let resp = client.request(req("")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // the error we return by default are *not* xml + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html; charset=utf-8" + ); + let result = String::from_utf8( + resp.into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(), + ) + .unwrap(); + assert!(result.contains("not found")); +} diff --git a/src/k2v-client/Cargo.toml b/src/k2v-client/Cargo.toml index 52c16d89..bbd09b19 100644 --- a/src/k2v-client/Cargo.toml +++ b/src/k2v-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "k2v-client" -version = "0.1.1" +version = "0.0.4" authors = ["Trinity Pointard ", "Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -9,25 +9,31 @@ repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" [dependencies] -base64 = "0.21" -http = "0.2" -log = "0.4" -rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] } -rusoto_credential = "0.48.0" -rusoto_signature = "0.48.0" -hyper-rustls = { version = "0.23", default-features = false, features = [ "http1", "http2", "tls12" ] } -serde = "1.0" -serde_json = "1.0" -thiserror = "1.0" -tokio = "1.24" +base64.workspace = true +sha2.workspace = true +hex.workspace = true +http.workspace = true +http-body-util.workspace = true +log.workspace = true +aws-sigv4.workspace = true +aws-sdk-config.workspace = true +percent-encoding.workspace = true +hyper = { workspace = true, default-features = false, features = ["http1", "http2"] } +hyper-util.workspace = true +hyper-rustls.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true # cli deps -clap = { version = "4.1", optional = true, features = ["derive", "env"] } -garage_util = { version = "0.8.2", path = "../util", optional = true } +clap = { workspace = true, optional = true } +format_table = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, optional = true } [features] -cli = ["clap", "tokio/fs", "tokio/io-std", "garage_util"] +cli = ["clap", "tokio/fs", "tokio/io-std", "tracing-subscriber", "format_table"] [lib] path = "lib.rs" diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs index cdd63cce..b1c2169b 100644 --- a/src/k2v-client/bin/k2v-cli.rs +++ b/src/k2v-client/bin/k2v-cli.rs @@ -2,12 +2,11 @@ use std::collections::BTreeMap; use std::process::exit; use std::time::Duration; +use base64::prelude::*; + use k2v_client::*; -use garage_util::formater::format_table; - -use rusoto_core::credential::AwsCredentials; -use rusoto_core::Region; +use format_table::format_table; use clap::{Parser, Subcommand}; @@ -55,7 +54,7 @@ enum Command { partition_key: String, /// Sort key to read from sort_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: ReadOutputKind, }, @@ -71,7 +70,7 @@ enum Command { /// Timeout, in seconds #[clap(short = 'T', long)] timeout: Option, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: ReadOutputKind, }, @@ -88,7 +87,7 @@ enum Command { /// Timeout, in seconds #[clap(short = 'T', long)] timeout: Option, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, }, @@ -104,7 +103,7 @@ enum Command { }, /// List partition keys ReadIndex { - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Output only partition keys matching this filter @@ -115,7 +114,7 @@ enum Command { ReadRange { /// Partition key to read from partition_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Output only sort keys matching this filter @@ -126,7 +125,7 @@ enum Command { DeleteRange { /// Partition key to delete from partition_key: String, - /// Output formating + /// Output formatting #[clap(flatten)] output_kind: BatchOutputKind, /// Delete only sort keys matching this filter @@ -155,7 +154,9 @@ impl Value { if let Some(ref text) = self.text { Ok(text.as_bytes().to_vec()) } else if let Some(ref b64) = self.b64 { - base64::decode(b64).map_err(|_| Error::Message("invalid base64 input".into())) + BASE64_STANDARD + .decode(b64) + .map_err(|_| Error::Message("invalid base64 input".into())) } else if let Some(ref path) = self.file { use tokio::io::AsyncReadExt; if path == "-" { @@ -184,10 +185,10 @@ struct ReadOutputKind { /// Raw output. Conflicts generate error, causality token is not returned #[clap(short, long, group = "output-kind")] raw: bool, - /// Human formated output + /// Human formatted output #[clap(short = 'H', long, group = "output-kind")] human: bool, - /// JSON formated output + /// JSON formatted output #[clap(short, long, group = "output-kind")] json: bool, } @@ -206,7 +207,7 @@ impl ReadOutputKind { let mut val = val.value; if val.len() != 1 { eprintln!( - "Raw mode can only read non-concurent values, found {} values, expected 1", + "Raw mode can only read non-concurrent values, found {} values, expected 1", val.len() ); exit(1); @@ -230,7 +231,7 @@ impl ReadOutputKind { for val in val.value { match val { K2vValue::Value(v) => { - println!("{}", base64::encode(&v)) + println!("{}", BASE64_STANDARD.encode(&v)) } K2vValue::Tombstone => { println!(); @@ -249,7 +250,7 @@ impl ReadOutputKind { if let Ok(string) = std::str::from_utf8(&v) { println!(" utf-8: {}", string); } else { - println!(" base64: {}", base64::encode(&v)); + println!(" base64: {}", BASE64_STANDARD.encode(&v)); } } K2vValue::Tombstone => { @@ -264,10 +265,10 @@ impl ReadOutputKind { #[derive(Parser, Debug)] #[clap(group = clap::ArgGroup::new("output-kind").multiple(false).required(false))] struct BatchOutputKind { - /// Human formated output + /// Human formatted output #[clap(short = 'H', long, group = "output-kind")] human: bool, - /// JSON formated output + /// JSON formatted output #[clap(short, long, group = "output-kind")] json: bool, } @@ -275,7 +276,7 @@ struct BatchOutputKind { impl BatchOutputKind { fn display_human_output(&self, values: BTreeMap) -> ! { for (key, values) in values { - println!("key: {}", key); + println!("sort_key: {}", key); let causality: String = values.causality.into(); println!("causality: {}", causality); for value in values.value { @@ -284,7 +285,7 @@ impl BatchOutputKind { if let Ok(string) = std::str::from_utf8(&v) { println!(" value(utf-8): {}", string); } else { - println!(" value(base64): {}", base64::encode(&v)); + println!(" value(base64): {}", BASE64_STANDARD.encode(&v)); } } K2vValue::Tombstone => { @@ -310,23 +311,19 @@ impl BatchOutputKind { .collect::>() } - fn display_poll_range_output( - &self, - seen_marker: String, - values: BTreeMap, - ) -> ! { + fn display_poll_range_output(&self, poll_range: PollRangeResult) -> ! { if self.json { let json = serde_json::json!({ - "values": self.values_json(values), - "seen_marker": seen_marker, + "values": self.values_json(poll_range.items), + "seen_marker": poll_range.seen_marker, }); let stdout = std::io::stdout(); serde_json::to_writer_pretty(stdout, &json).unwrap(); exit(0) } else { - println!("seen marker: {}", seen_marker); - self.display_human_output(values) + println!("seen marker: {}", poll_range.seen_marker); + self.display_human_output(poll_range.items) } } @@ -393,16 +390,27 @@ impl Filter { #[tokio::main] async fn main() -> Result<(), Error> { + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "warn") + } + + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) + .init(); + let args = Args::parse(); - let region = Region::Custom { - name: args.region, + let config = K2vClientConfig { endpoint: args.endpoint, + region: args.region, + aws_access_key_id: args.key_id, + aws_secret_access_key: args.secret, + bucket: args.bucket, + user_agent: None, }; - let creds = AwsCredentials::new(args.key_id, args.secret, None, None); - - let client = K2vClient::new(region, args.bucket, creds, None)?; + let client = K2vClient::new(config)?; match args.command { Command::Insert { @@ -489,8 +497,8 @@ async fn main() -> Result<(), Error> { ) .await?; match res { - Some((items, seen_marker)) => { - output_kind.display_poll_range_output(seen_marker, items); + Some(poll_range_output) => { + output_kind.display_poll_range_output(poll_range_output); } None => { if output_kind.json { @@ -520,7 +528,7 @@ async fn main() -> Result<(), Error> { value .as_object_mut() .unwrap() - .insert("sort_key".to_owned(), k.into()); + .insert("partition_key".to_owned(), k.into()); value }) .collect::>(); @@ -537,7 +545,7 @@ async fn main() -> Result<(), Error> { } let mut to_print = Vec::new(); - to_print.push(format!("key:\tentries\tconflicts\tvalues\tbytes")); + to_print.push(format!("partition_key\tentries\tconflicts\tvalues\tbytes")); for (k, v) in res.items { to_print.push(format!( "{}\t{}\t{}\t{}\t{}", diff --git a/src/k2v-client/error.rs b/src/k2v-client/error.rs index 37c221f2..96f5674a 100644 --- a/src/k2v-client/error.rs +++ b/src/k2v-client/error.rs @@ -18,12 +18,22 @@ pub enum Error { NotFound, #[error("io error: {0}")] IoError(#[from] std::io::Error), - #[error("rusoto tls error: {0}")] - RusotoTls(#[from] rusoto_core::request::TlsError), - #[error("rusoto http error: {0}")] - RusotoHttp(#[from] rusoto_core::HttpDispatchError), + #[error("http error: {0}")] + Http(#[from] http::Error), + #[error("hyper error: {0}")] + Hyper(#[from] hyper::Error), + #[error("hyper client error: {0}")] + HyperClient(#[from] hyper_util::client::legacy::Error), + #[error("invalid header: {0}")] + Header(#[from] hyper::header::ToStrError), #[error("deserialization error: {0}")] Deserialization(#[from] serde_json::Error), + #[error("invalid signature parameters: {0}")] + SignParameters(#[from] aws_sigv4::sign::v4::signing_params::BuildError), + #[error("could not sign request: {0}")] + SignRequest(#[from] aws_sigv4::http_request::SigningError), + #[error("request timed out")] + Timeout, #[error("{0}")] Message(Cow<'static, str>), } diff --git a/src/k2v-client/lib.rs b/src/k2v-client/lib.rs index ca52d0cf..fe8fd3e0 100644 --- a/src/k2v-client/lib.rs +++ b/src/k2v-client/lib.rs @@ -1,62 +1,97 @@ use std::collections::BTreeMap; -use std::time::Duration; +use std::convert::TryInto; +use std::time::{Duration, SystemTime}; -use http::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE}; -use http::status::StatusCode; -use http::HeaderMap; +use base64::prelude::*; use log::{debug, error}; +use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC}; + +use http::header::{ACCEPT, CONTENT_TYPE}; +use http::status::StatusCode; +use http::{HeaderName, HeaderValue, Request}; +use http_body_util::{BodyExt, Full as FullBody}; +use hyper::body::Bytes; +use hyper_rustls::HttpsConnector; +use hyper_util::client::legacy::{connect::HttpConnector, Client as HttpClient}; +use hyper_util::rt::TokioExecutor; + +use aws_sdk_config::config::Credentials; +use aws_sigv4::http_request::{sign, SignableBody, SignableRequest, SigningSettings}; +use aws_sigv4::sign::v4::SigningParams; -use rusoto_core::{ByteStream, DispatchSignedRequest, HttpClient}; -use rusoto_credential::AwsCredentials; -use rusoto_signature::region::Region; -use rusoto_signature::signature::SignedRequest; use serde::de::Error as DeError; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use tokio::io::AsyncReadExt; - mod error; pub use error::Error; +pub type Body = FullBody; + const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_POLL_TIMEOUT: Duration = Duration::from_secs(300); const SERVICE: &str = "k2v"; -const GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; +const AMZ_CONTENT_SHA256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); +const GARAGE_CAUSALITY_TOKEN: HeaderName = HeaderName::from_static("x-garage-causality-token"); + +const STRICT_ENCODE_SET: AsciiSet = NON_ALPHANUMERIC + .remove(b'_') + .remove(b'-') + .remove(b'.') + .remove(b'~'); +const PATH_ENCODE_SET: AsciiSet = NON_ALPHANUMERIC + .remove(b'/') + .remove(b'_') + .remove(b'-') + .remove(b'.') + .remove(b'~'); + +pub struct K2vClientConfig { + pub endpoint: String, + pub region: String, + pub aws_access_key_id: String, + pub aws_secret_access_key: String, + pub bucket: String, + pub user_agent: Option, +} /// Client used to query a K2V server. pub struct K2vClient { - region: Region, - bucket: String, - creds: AwsCredentials, - client: HttpClient, + config: K2vClientConfig, + user_agent: HeaderValue, + client: HttpClient, Body>, } impl K2vClient { /// Create a new K2V client. - pub fn new( - region: Region, - bucket: String, - creds: AwsCredentials, - user_agent: Option, - ) -> Result { + pub fn new(config: K2vClientConfig) -> Result { let connector = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() + .with_native_roots()? .https_or_http() .enable_http1() .enable_http2() .build(); - let mut client = HttpClient::from_connector(connector); - if let Some(ua) = user_agent { - client.local_agent_prepend(ua); - } else { - client.local_agent_prepend(format!("k2v/{}", env!("CARGO_PKG_VERSION"))); - } + let client = HttpClient::builder(TokioExecutor::new()).build(connector); + Self::new_with_client(config, client) + } + + /// Create a new K2V client with an external client. + /// Useful for example if you plan on creating many clients but you want to mutualize the + /// underlying thread pools & co. + pub fn new_with_client( + config: K2vClientConfig, + client: HttpClient, Body>, + ) -> Result { + let user_agent: std::borrow::Cow = match &config.user_agent { + Some(ua) => ua.into(), + None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(), + }; + let user_agent = HeaderValue::from_str(&user_agent) + .map_err(|_| Error::Message("invalid user agent".into()))?; Ok(K2vClient { - region, - bucket, - creds, + config, client, + user_agent, }) } @@ -66,15 +101,10 @@ impl K2vClient { partition_key: &str, sort_key: &str, ) -> Result { - let mut req = SignedRequest::new( - "GET", - SERVICE, - &self.region, - &format!("/{}/{}", self.bucket, partition_key), - ); - req.add_param("sort_key", sort_key); - req.add_header(ACCEPT, "application/octet-stream, application/json"); - + let url = self.build_url(Some(partition_key), &[("sort_key", sort_key)]); + let req = Request::get(url) + .header(ACCEPT, "application/octet-stream, application/json") + .body(Bytes::new())?; let res = self.dispatch(req, None).await?; let causality = res @@ -91,7 +121,7 @@ impl K2vClient { match res.content_type.as_deref() { Some("application/octet-stream") => Ok(CausalValue { causality, - value: vec![K2vValue::Value(res.body)], + value: vec![K2vValue::Value(res.body.to_vec())], }), Some("application/json") => { let value = serde_json::from_slice(&res.body)?; @@ -115,16 +145,17 @@ impl K2vClient { ) -> Result, Error> { let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT); - let mut req = SignedRequest::new( - "GET", - SERVICE, - &self.region, - &format!("/{}/{}", self.bucket, partition_key), + let url = self.build_url( + Some(partition_key), + &[ + ("sort_key", sort_key), + ("causality_token", &causality.0), + ("timeout", &timeout.as_secs().to_string()), + ], ); - req.add_param("sort_key", sort_key); - req.add_param("causality_token", &causality.0); - req.add_param("timeout", &timeout.as_secs().to_string()); - req.add_header(ACCEPT, "application/octet-stream, application/json"); + let req = Request::get(url) + .header(ACCEPT, "application/octet-stream, application/json") + .body(Bytes::new())?; let res = self.dispatch(req, Some(timeout + DEFAULT_TIMEOUT)).await?; @@ -146,7 +177,7 @@ impl K2vClient { match res.content_type.as_deref() { Some("application/octet-stream") => Ok(Some(CausalValue { causality, - value: vec![K2vValue::Value(res.body)], + value: vec![K2vValue::Value(res.body.to_vec())], })), Some("application/json") => { let value = serde_json::from_slice(&res.body)?; @@ -167,7 +198,7 @@ impl K2vClient { filter: Option>, seen_marker: Option<&str>, timeout: Option, - ) -> Result, String)>, Error> { + ) -> Result, Error> { let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT); let request = PollRangeRequest { @@ -176,16 +207,10 @@ impl K2vClient { timeout: timeout.as_secs(), }; - let mut req = SignedRequest::new( - "POST", - SERVICE, - &self.region, - &format!("/{}/{}", self.bucket, partition_key), - ); - req.add_param("poll_range", ""); - + let url = self.build_url(Some(partition_key), &[("poll_range", "")]); let payload = serde_json::to_vec(&request)?; - req.set_payload(Some(payload)); + let req = Request::post(url).body(Bytes::from(payload))?; + let res = self.dispatch(req, Some(timeout + DEFAULT_TIMEOUT)).await?; if res.status == StatusCode::NOT_MODIFIED { @@ -208,7 +233,10 @@ impl K2vClient { }) .collect::>(); - Ok(Some((items, resp.seen_marker))) + Ok(Some(PollRangeResult { + items, + seen_marker: resp.seen_marker, + })) } /// Perform an InsertItem request, inserting a value for a single pk+sk. @@ -219,18 +247,12 @@ impl K2vClient { value: Vec, causality: Option, ) -> Result<(), Error> { - let mut req = SignedRequest::new( - "PUT", - SERVICE, - &self.region, - &format!("/{}/{}", self.bucket, partition_key), - ); - req.add_param("sort_key", sort_key); - req.set_payload(Some(value)); - + let url = self.build_url(Some(partition_key), &[("sort_key", sort_key)]); + let mut req = Request::put(url); if let Some(causality) = causality { - req.add_header(GARAGE_CAUSALITY_TOKEN, &causality.0); + req = req.header(GARAGE_CAUSALITY_TOKEN, &causality.0); } + let req = req.body(Bytes::from(value))?; self.dispatch(req, None).await?; Ok(()) @@ -243,14 +265,10 @@ impl K2vClient { sort_key: &str, causality: CausalityToken, ) -> Result<(), Error> { - let mut req = SignedRequest::new( - "DELETE", - SERVICE, - &self.region, - &format!("/{}/{}", self.bucket, partition_key), - ); - req.add_param("sort_key", sort_key); - req.add_header(GARAGE_CAUSALITY_TOKEN, &causality.0); + let url = self.build_url(Some(partition_key), &[("sort_key", sort_key)]); + let req = Request::delete(url) + .header(GARAGE_CAUSALITY_TOKEN, &causality.0) + .body(Bytes::new())?; self.dispatch(req, None).await?; Ok(()) @@ -262,9 +280,9 @@ impl K2vClient { &self, filter: Filter<'_>, ) -> Result, Error> { - let mut req = - SignedRequest::new("GET", SERVICE, &self.region, &format!("/{}", self.bucket)); - filter.insert_params(&mut req); + let params = filter.query_params(); + let url = self.build_url(None, ¶ms); + let req = Request::get(url).body(Bytes::new())?; let res = self.dispatch(req, None).await?; @@ -286,11 +304,10 @@ impl K2vClient { /// *not* atomic: it is possible for some sub-operations to fails and others to success. In /// that case, failure is reported. pub async fn insert_batch(&self, operations: &[BatchInsertOp<'_>]) -> Result<(), Error> { - let mut req = - SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket)); - + let url = self.build_url::<&str>(None, &[]); let payload = serde_json::to_vec(operations)?; - req.set_payload(Some(payload)); + let req = Request::post(url).body(payload.into())?; + self.dispatch(req, None).await?; Ok(()) } @@ -300,12 +317,10 @@ impl K2vClient { &self, operations: &[BatchReadOp<'_>], ) -> Result>, Error> { - let mut req = - SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket)); - req.add_param("search", ""); - + let url = self.build_url(None, &[("search", "")]); let payload = serde_json::to_vec(operations)?; - req.set_payload(Some(payload)); + let req = Request::post(url).body(payload.into())?; + let res = self.dispatch(req, None).await?; let resp: Vec = serde_json::from_slice(&res.body)?; @@ -331,15 +346,13 @@ impl K2vClient { .collect()) } - /// Perform a DeleteBatch request, deleting mutiple values or range of values at once, without + /// Perform a DeleteBatch request, deleting multiple values or range of values at once, without /// providing causality information. pub async fn delete_batch(&self, operations: &[BatchDeleteOp<'_>]) -> Result, Error> { - let mut req = - SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket)); - req.add_param("delete", ""); - + let url = self.build_url(None, &[("delete", "")]); let payload = serde_json::to_vec(operations)?; - req.set_payload(Some(payload)); + let req = Request::post(url).body(payload.into())?; + let res = self.dispatch(req, None).await?; let resp: Vec = serde_json::from_slice(&res.body)?; @@ -349,33 +362,87 @@ impl K2vClient { async fn dispatch( &self, - mut req: SignedRequest, + mut req: Request, timeout: Option, ) -> Result { - req.sign(&self.creds); - let mut res = self - .client - .dispatch(req, Some(timeout.unwrap_or(DEFAULT_TIMEOUT))) - .await?; + req.headers_mut() + .insert(http::header::USER_AGENT, self.user_agent.clone()); - let causality_token = res - .headers - .remove(GARAGE_CAUSALITY_TOKEN) - .map(CausalityToken); - let content_type = res.headers.remove(CONTENT_TYPE); + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(req.body()); + let hash = hex::encode(&hasher.finalize()); + req.headers_mut() + .insert(AMZ_CONTENT_SHA256, hash.try_into().unwrap()); + + debug!("request uri: {:?}", req.uri()); + + // Sign request + let signing_settings = SigningSettings::default(); + let identity = Credentials::new( + &self.config.aws_access_key_id, + &self.config.aws_secret_access_key, + None, + None, + "k2v-client", + ) + .into(); + let signing_params = SigningParams::builder() + .identity(&identity) + .region(&self.config.region) + .name(SERVICE) + .time(SystemTime::now()) + .settings(signing_settings) + .build()? + .into(); + // Convert the HTTP request into a signable request + let signable_request = SignableRequest::new( + req.method().as_str(), + req.uri().to_string(), + // TODO: get rid of Unwrap + req.headers() + .iter() + .map(|(x, y)| (x.as_str(), y.to_str().unwrap())), + SignableBody::Bytes(req.body().as_ref()), + )?; + + // Sign and then apply the signature to the request + let (signing_instructions, _signature) = + sign(signable_request, &signing_params)?.into_parts(); + signing_instructions.apply_to_request_http1x(&mut req); + + // Send and wait for timeout + let res = tokio::select! { + res = self.client.request(req.map(Body::from)) => res?, + _ = tokio::time::sleep(timeout.unwrap_or(DEFAULT_TIMEOUT)) => { + return Err(Error::Timeout); + } + }; + + let (mut res, body) = res.into_parts(); + let causality_token = match res.headers.remove(GARAGE_CAUSALITY_TOKEN) { + Some(v) => Some(CausalityToken(v.to_str()?.to_string())), + None => None, + }; + let content_type = match res.headers.remove(CONTENT_TYPE) { + Some(v) => Some(v.to_str()?.to_string()), + None => None, + }; let body = match res.status { - StatusCode::OK => read_body(&mut res.headers, res.body).await?, - StatusCode::NO_CONTENT => Vec::new(), + StatusCode::OK => BodyExt::collect(body).await?.to_bytes(), + StatusCode::NO_CONTENT => Bytes::new(), StatusCode::NOT_FOUND => return Err(Error::NotFound), - StatusCode::NOT_MODIFIED => Vec::new(), + StatusCode::NOT_MODIFIED => Bytes::new(), s => { - let err_body = read_body(&mut res.headers, res.body) + let err_body = body + .collect() .await + .map(|x| x.to_bytes()) .unwrap_or_default(); let err_body_str = std::str::from_utf8(&err_body) .map(String::from) - .unwrap_or_else(|_| base64::encode(&err_body)); + .unwrap_or_else(|_| BASE64_STANDARD.encode(&err_body)); if s.is_client_error() || s.is_server_error() { error!("Error response {}: {}", res.status, err_body_str); @@ -408,7 +475,7 @@ impl K2vClient { "Response body: {}", std::str::from_utf8(&body) .map(String::from) - .unwrap_or_else(|_| base64::encode(&body)) + .unwrap_or_else(|_| BASE64_STANDARD.encode(&body)) ); Ok(Response { @@ -418,16 +485,30 @@ impl K2vClient { content_type, }) } -} -async fn read_body(headers: &mut HeaderMap, body: ByteStream) -> Result, Error> { - let body_len = headers - .get(CONTENT_LENGTH) - .and_then(|h| h.parse().ok()) - .unwrap_or(0); - let mut res = Vec::with_capacity(body_len); - body.into_async_read().read_to_end(&mut res).await?; - Ok(res) + fn build_url>(&self, partition_key: Option<&str>, query: &[(&str, V)]) -> String { + let mut url = format!( + "{}/{}", + self.config.endpoint.trim_end_matches('/'), + self.config.bucket + ); + if let Some(pk) = partition_key { + url.push('/'); + url.extend(utf8_percent_encode(pk, &PATH_ENCODE_SET)); + } + if !query.is_empty() { + url.push('?'); + for (i, (k, v)) in query.iter().enumerate() { + if i > 0 { + url.push('&'); + } + url.extend(utf8_percent_encode(k, &STRICT_ENCODE_SET)); + url.push('='); + url.extend(utf8_percent_encode(v.as_ref(), &STRICT_ENCODE_SET)); + } + } + url + } } /// An opaque token used to convey causality between operations. @@ -482,9 +563,11 @@ impl<'de> Deserialize<'de> for K2vValue { { let val: Option<&str> = Option::deserialize(d)?; Ok(match val { - Some(s) => { - K2vValue::Value(base64::decode(s).map_err(|_| DeError::custom("invalid base64"))?) - } + Some(s) => K2vValue::Value( + BASE64_STANDARD + .decode(s) + .map_err(|_| DeError::custom("invalid base64"))?, + ), None => K2vValue::Tombstone, }) } @@ -498,7 +581,7 @@ impl Serialize for K2vValue { match self { K2vValue::Tombstone => serializer.serialize_none(), K2vValue::Value(v) => { - let b64 = base64::encode(v); + let b64 = BASE64_STANDARD.encode(v); serializer.serialize_str(&b64) } } @@ -530,6 +613,7 @@ pub struct Filter<'a> { pub reverse: bool, } +/// Filter for a poll range operations. #[derive(Debug, Default, Clone, Serialize)] pub struct PollRangeFilter<'a> { pub start: Option<&'a str>, @@ -537,6 +621,15 @@ pub struct PollRangeFilter<'a> { pub prefix: Option<&'a str>, } +/// Response to a poll_range query +#[derive(Debug, Default, Clone, Serialize)] +pub struct PollRangeResult { + /// List of items that have changed since last PollRange call. + pub items: BTreeMap, + /// opaque string representing items already seen for future PollRange calls. + pub seen_marker: String, +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] struct PollRangeRequest<'a> { @@ -554,22 +647,24 @@ struct PollRangeResponse { } impl<'a> Filter<'a> { - fn insert_params(&self, req: &mut SignedRequest) { - if let Some(start) = &self.start { - req.add_param("start", start); + fn query_params(&self) -> Vec<(&'static str, std::borrow::Cow)> { + let mut res = Vec::<(&'static str, std::borrow::Cow)>::with_capacity(8); + if let Some(start) = self.start.as_deref() { + res.push(("start", start.into())); } - if let Some(end) = &self.end { - req.add_param("end", end); + if let Some(end) = self.end.as_deref() { + res.push(("end", end.into())); } - if let Some(prefix) = &self.prefix { - req.add_param("prefix", prefix); + if let Some(prefix) = self.prefix.as_deref() { + res.push(("prefix", prefix.into())); } if let Some(limit) = &self.limit { - req.add_param("limit", &limit.to_string()); + res.push(("limit", limit.to_string().into())); } if self.reverse { - req.add_param("reverse", "true"); + res.push(("reverse", "true".into())); } + res } } @@ -691,7 +786,7 @@ struct ErrorResponse { } struct Response { - body: Vec, + body: Bytes, status: StatusCode, causality_token: Option, content_type: Option, diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 2b525a42..289c0024 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,35 +14,34 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.2", default-features = false, path = "../db" } -garage_rpc = { version = "0.8.2", path = "../rpc" } -garage_table = { version = "0.8.2", path = "../table" } -garage_block = { version = "0.8.2", path = "../block" } -garage_util = { version = "0.8.2", path = "../util" } +garage_db.workspace = true +garage_rpc.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_util.workspace = true +garage_net.workspace = true -async-trait = "0.1.7" -arc-swap = "1.0" -blake2 = "0.10" -err-derive = "0.3" -hex = "0.4" -base64 = "0.21" -tracing = "0.1" -rand = "0.8" -zstd = { version = "0.12", default-features = false } +async-trait.workspace = true +blake2.workspace = true +chrono.workspace = true +thiserror.workspace = true +hex.workspace = true +http.workspace = true +base64.workspace = true +parse_duration.workspace = true +tracing.workspace = true +rand.workspace = true +zstd.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true +serde_bytes.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -opentelemetry = "0.17" - -netapp = "0.5" +futures.workspace = true +tokio.workspace = true [features] -default = [ "sled" ] +default = [ "lmdb", "sqlite" ] k2v = [ "garage_util/k2v" ] lmdb = [ "garage_db/lmdb" ] -sled = [ "garage_db/sled" ] sqlite = [ "garage_db/sqlite" ] +fjall = [ "garage_db/fjall" ] diff --git a/src/model/bucket_alias_table.rs b/src/model/bucket_alias_table.rs index 54d7fbad..276d0d1c 100644 --- a/src/model/bucket_alias_table.rs +++ b/src/model/bucket_alias_table.rs @@ -22,14 +22,10 @@ mod v08 { pub use v08::*; impl BucketAlias { - pub fn new(name: String, ts: u64, bucket_id: Option) -> Option { - if !is_valid_bucket_name(&name) { - None - } else { - Some(BucketAlias { - name, - state: crdt::Lww::raw(ts, bucket_id), - }) + pub fn new(name: String, ts: u64, bucket_id: Option) -> Self { + BucketAlias { + name, + state: crdt::Lww::raw(ts, bucket_id), } } @@ -80,7 +76,7 @@ impl TableSchema for BucketAliasTable { /// In the case of Garage, bucket names must not be hex-encoded /// 32 byte string, which is excluded thanks to the /// maximum length of 63 bytes given in the spec. -pub fn is_valid_bucket_name(n: &str) -> bool { +pub fn is_valid_bucket_name(n: &str, puny: bool) -> bool { // Bucket names must be between 3 and 63 characters n.len() >= 3 && n.len() <= 63 // Bucket names must be composed of lowercase letters, numbers, @@ -89,10 +85,12 @@ pub fn is_valid_bucket_name(n: &str) -> bool { // Bucket names must start and end with a letter or a number && !n.starts_with(&['-', '.'][..]) && !n.ends_with(&['-', '.'][..]) - // Bucket names must not be formated as an IP address + // Bucket names must not be formatted as an IP address && n.parse::().is_err() - // Bucket names must not start wih "xn--" - && !n.starts_with("xn--") + // Bucket names must not start with "xn--" + && (!n.starts_with("xn--") || puny) + // We are a bit stricter, to properly restrict punycode in all labels + && (!n.contains(".xn--") || puny) // Bucket names must not end with "-s3alias" && !n.ends_with("-s3alias") } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index ac163736..f1cc032e 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -14,7 +14,7 @@ mod v08 { /// A bucket is a collection of objects /// /// Its parameters are not directly accessible as: - /// - It must be possible to merge paramaters, hence the use of a LWW CRDT. + /// - It must be possible to merge parameters, hence the use of a LWW CRDT. /// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Bucket { @@ -48,6 +48,9 @@ mod v08 { pub website_config: crdt::Lww>, /// CORS rules pub cors_config: crdt::Lww>>, + /// Lifecycle configuration + #[serde(default)] + pub lifecycle_config: crdt::Lww>>, /// Bucket quotas #[serde(default)] pub quotas: crdt::Lww, @@ -69,6 +72,42 @@ mod v08 { pub expose_headers: Vec, } + /// Lifecycle configuration rule + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct LifecycleRule { + /// The ID of the rule + pub id: Option, + /// Whether the rule is active + pub enabled: bool, + /// The filter to check whether rule applies to a given object + pub filter: LifecycleFilter, + /// Number of days after which incomplete multipart uploads are aborted + pub abort_incomplete_mpu_days: Option, + /// Expiration policy for stored objects + pub expiration: Option, + } + + /// A lifecycle filter is a set of conditions that must all be true. + /// For each condition, if it is None, it is not verified (always true), + /// and if it is Some(x), then it is verified for value x + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Default)] + pub struct LifecycleFilter { + /// If Some(x), object key has to start with prefix x + pub prefix: Option, + /// If Some(x), object size has to be more than x + pub size_gt: Option, + /// If Some(x), object size has to be less than x + pub size_lt: Option, + } + + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub enum LifecycleExpiration { + /// Objects expire x days after they were created + AfterDays(usize), + /// Objects expire at date x (must be in yyyy-mm-dd format) + AtDate(String), + } + #[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] pub struct BucketQuotas { /// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket) @@ -87,8 +126,8 @@ impl AutoCrdt for BucketQuotas { } impl BucketParams { - /// Create an empty BucketParams with no authorized keys and no website accesss - pub fn new() -> Self { + /// Create an empty BucketParams with no authorized keys and no website access + fn new() -> Self { BucketParams { creation_date: now_msec(), authorized_keys: crdt::Map::new(), @@ -96,6 +135,7 @@ impl BucketParams { local_aliases: crdt::LwwMap::new(), website_config: crdt::Lww::new(None), cors_config: crdt::Lww::new(None), + lifecycle_config: crdt::Lww::new(None), quotas: crdt::Lww::new(BucketQuotas::default()), } } @@ -111,10 +151,25 @@ impl Crdt for BucketParams { self.website_config.merge(&o.website_config); self.cors_config.merge(&o.cors_config); + self.lifecycle_config.merge(&o.lifecycle_config); self.quotas.merge(&o.quotas); } } +pub fn parse_lifecycle_date(date: &str) -> Result { + use chrono::prelude::*; + + if let Ok(datetime) = NaiveDateTime::parse_from_str(date, "%Y-%m-%dT%H:%M:%SZ") { + if datetime.time() == NaiveTime::MIN { + Ok(datetime.date()) + } else { + Err("date must be at midnight") + } + } else { + NaiveDate::parse_from_str(date, "%Y-%m-%d").map_err(|_| "date has invalid format") + } +} + impl Default for Bucket { fn default() -> Self { Self::new() @@ -136,6 +191,13 @@ impl Bucket { } } + pub fn present(id: Uuid, params: BucketParams) -> Self { + Bucket { + id, + state: crdt::Deletable::present(params), + } + } + /// Returns true if this represents a deleted bucket pub fn is_deleted(&self) -> bool { self.state.is_deleted() diff --git a/src/model/garage.rs b/src/model/garage.rs index 3daa1b33..f4f6f693 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -1,14 +1,16 @@ +use std::str::FromStr; use std::sync::Arc; -use netapp::NetworkKey; +use garage_net::NetworkKey; use garage_db as db; use garage_util::background::*; use garage_util::config::*; use garage_util::error::*; +use garage_util::persister::PersisterShared; -use garage_rpc::replication_mode::ReplicationMode; +use garage_rpc::replication_mode::*; use garage_rpc::system::System; use garage_block::manager::*; @@ -17,6 +19,8 @@ use garage_table::replication::TableShardedReplication; use garage_table::*; use crate::s3::block_ref_table::*; +use crate::s3::lifecycle_worker; +use crate::s3::mpu_table::*; use crate::s3::object_table::*; use crate::s3::version_table::*; @@ -36,8 +40,8 @@ pub struct Garage { /// The set of background variables that can be viewed/modified at runtime pub bg_vars: vars::BgVars, - /// The replication mode of this cluster - pub replication_mode: ReplicationMode, + /// The replication factor of this cluster + pub replication_factor: ReplicationFactor, /// The local database pub db: db::Db, @@ -53,15 +57,25 @@ pub struct Garage { /// Table containing api keys pub key_table: Arc>, + /// Lock to prevent concurrent modification of buckets and access keys + bucket_lock: tokio::sync::Mutex<()>, + /// Table containing S3 objects pub object_table: Arc>, /// Counting table containing object counters pub object_counter_table: Arc>, + /// Table containing S3 multipart uploads + pub mpu_table: Arc>, + /// Counting table containing multipart object counters + pub mpu_counter_table: Arc>, /// Table containing S3 object versions pub version_table: Arc>, /// Table containing S3 block references (not blocks themselves) pub block_ref_table: Arc>, + /// Persister for lifecycle worker info + pub lifecycle_persister: PersisterShared, + #[cfg(feature = "k2v")] pub k2v: GarageK2V, } @@ -82,132 +96,78 @@ impl Garage { // Create meta dir and data dir if they don't exist already std::fs::create_dir_all(&config.metadata_dir) .ok_or_message("Unable to create Garage metadata directory")?; - std::fs::create_dir_all(&config.data_dir) - .ok_or_message("Unable to create Garage data directory")?; + match &config.data_dir { + DataDirEnum::Single(data_dir) => { + std::fs::create_dir_all(data_dir).ok_or_message(format!( + "Unable to create Garage data directory: {}", + data_dir.to_string_lossy() + ))?; + } + DataDirEnum::Multiple(data_dirs) => { + for dir in data_dirs { + std::fs::create_dir_all(&dir.path).ok_or_message(format!( + "Unable to create Garage data directory: {}", + dir.path.to_string_lossy() + ))?; + } + } + } info!("Opening database..."); - let mut db_path = config.metadata_dir.clone(); - let db = match config.db_engine.as_str() { - // ---- Sled DB ---- - #[cfg(feature = "sled")] - "sled" => { - db_path.push("db"); - info!("Opening Sled database at: {}", db_path.display()); - let db = db::sled_adapter::sled::Config::default() - .path(&db_path) - .cache_capacity(config.sled_cache_capacity) - .flush_every_ms(Some(config.sled_flush_every_ms)) - .open() - .ok_or_message("Unable to open sled DB")?; - db::sled_adapter::SledDb::init(db) - } - #[cfg(not(feature = "sled"))] - "sled" => return Err(Error::Message("sled db not available in this build".into())), - // ---- Sqlite DB ---- - #[cfg(feature = "sqlite")] - "sqlite" | "sqlite3" | "rusqlite" => { - db_path.push("db.sqlite"); - info!("Opening Sqlite database at: {}", db_path.display()); - let db = db::sqlite_adapter::rusqlite::Connection::open(db_path) - .ok_or_message("Unable to open sqlite DB")?; - db::sqlite_adapter::SqliteDb::init(db) - } - #[cfg(not(feature = "sqlite"))] - "sqlite" | "sqlite3" | "rusqlite" => { - return Err(Error::Message( - "sqlite db not available in this build".into(), - )) - } - // ---- LMDB DB ---- - #[cfg(feature = "lmdb")] - "lmdb" | "heed" => { - db_path.push("db.lmdb"); - info!("Opening LMDB database at: {}", db_path.display()); - std::fs::create_dir_all(&db_path) - .ok_or_message("Unable to create LMDB data directory")?; - let map_size = garage_db::lmdb_adapter::recommended_map_size(); - - use db::lmdb_adapter::heed; - let mut env_builder = heed::EnvOpenOptions::new(); - env_builder.max_dbs(100); - env_builder.max_readers(500); - env_builder.map_size(map_size); - unsafe { - env_builder.flag(heed::flags::Flags::MdbNoSync); - env_builder.flag(heed::flags::Flags::MdbNoMetaSync); - } - let db = match env_builder.open(&db_path) { - Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => { - return Err(Error::Message( - "OutOfMemory error while trying to open LMDB database. This can happen \ - if your operating system is not allowing you to use sufficient virtual \ - memory address space. Please check that no limit is set (ulimit -v). \ - On 32-bit machines, you should probably switch to another database engine.".into())) - } - x => x.ok_or_message("Unable to open LMDB DB")?, - }; - db::lmdb_adapter::LmdbDb::init(db) - } - #[cfg(not(feature = "lmdb"))] - "lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())), - // ---- Unavailable DB engine ---- - e => { - return Err(Error::Message(format!( - "Unsupported DB engine: {} (options: {})", - e, - vec![ - #[cfg(feature = "sled")] - "sled", - #[cfg(feature = "sqlite")] - "sqlite", - #[cfg(feature = "lmdb")] - "lmdb", - ] - .join(", ") - ))); - } + let db_engine = db::Engine::from_str(&config.db_engine) + .ok_or_message("Invalid `db_engine` value in configuration file")?; + let db_path = db_engine.db_path(&config.metadata_dir); + let db_opt = db::OpenOpt { + fsync: config.metadata_fsync, + lmdb_map_size: match config.lmdb_map_size { + v if v == usize::default() => None, + v => Some(v), + }, + fjall_block_cache_size: match config.fjall_block_cache_size { + v if v == usize::default() => None, + v => Some(v), + }, }; + let db = db::open_db(&db_path, db_engine, &db_opt) + .ok_or_message("Unable to open metadata db")?; + info!("Initializing RPC..."); let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message( "rpc_secret value is missing, not present in config file or in environment", )?) .ok() .and_then(|x| NetworkKey::from_slice(&x)) - .ok_or_message("Invalid RPC secret key")?; + .ok_or_message("Invalid RPC secret key: expected 32 bytes of random hex, please check the documentation for requirements")?; - let replication_mode = ReplicationMode::parse(&config.replication_mode) - .ok_or_message("Invalid replication_mode in config file.")?; + let (replication_factor, consistency_mode) = parse_replication_mode(&config)?; + + info!("Initialize background variable system..."); + let mut bg_vars = vars::BgVars::new(); info!("Initialize membership management system..."); - let system = System::new(network_key, replication_mode, &config)?; + let system = System::new(network_key, replication_factor, consistency_mode, &config)?; let data_rep_param = TableShardedReplication { system: system.clone(), - replication_factor: replication_mode.replication_factor(), - write_quorum: replication_mode.write_quorum(), + replication_factor: replication_factor.into(), + write_quorum: replication_factor.write_quorum(consistency_mode), read_quorum: 1, }; let meta_rep_param = TableShardedReplication { system: system.clone(), - replication_factor: replication_mode.replication_factor(), - write_quorum: replication_mode.write_quorum(), - read_quorum: replication_mode.read_quorum(), + replication_factor: replication_factor.into(), + write_quorum: replication_factor.write_quorum(consistency_mode), + read_quorum: replication_factor.read_quorum(consistency_mode), }; let control_rep_param = TableFullReplication { system: system.clone(), - max_faults: replication_mode.control_write_max_faults(), }; info!("Initialize block manager..."); - let block_manager = BlockManager::new( - &db, - config.data_dir.clone(), - config.compression_level, - data_rep_param, - system.clone(), - ); + let block_manager = BlockManager::new(&db, &config, data_rep_param, system.clone())?; + block_manager.register_bg_vars(&mut bg_vars); // ---- admin tables ---- info!("Initialize bucket_table..."); @@ -244,6 +204,20 @@ impl Garage { &db, ); + info!("Initialize multipart upload counter table..."); + let mpu_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db); + + info!("Initialize multipart upload table..."); + let mpu_table = Table::new( + MultipartUploadTable { + version_table: version_table.clone(), + mpu_counter_table: mpu_counter_table.clone(), + }, + meta_rep_param.clone(), + system.clone(), + &db, + ); + info!("Initialize object counter table..."); let object_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db); @@ -252,6 +226,7 @@ impl Garage { let object_table = Table::new( ObjectTable { version_table: version_table.clone(), + mpu_table: mpu_table.clone(), object_counter_table: object_counter_table.clone(), }, meta_rep_param.clone(), @@ -259,35 +234,48 @@ impl Garage { &db, ); + info!("Load lifecycle worker state..."); + let lifecycle_persister = + PersisterShared::new(&system.metadata_dir, "lifecycle_worker_state"); + lifecycle_worker::register_bg_vars(&lifecycle_persister, &mut bg_vars); + // ---- K2V ---- #[cfg(feature = "k2v")] let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param); - // Initialize bg vars - let mut bg_vars = vars::BgVars::new(); - block_manager.register_bg_vars(&mut bg_vars); + // ---- setup block refcount recalculation ---- + // this function can be used to fix inconsistencies in the RC table + block_manager.set_recalc_rc(vec![ + block_ref_recount_fn(&block_ref_table), + // other functions could be added here if we had other tables + // that hold references to data blocks + ]); // -- done -- Ok(Arc::new(Self { config, bg_vars, - replication_mode, + replication_factor, db, system, block_manager, bucket_table, bucket_alias_table, key_table, + bucket_lock: tokio::sync::Mutex::new(()), object_table, object_counter_table, + mpu_table, + mpu_counter_table, version_table, block_ref_table, + lifecycle_persister, #[cfg(feature = "k2v")] k2v, })) } - pub fn spawn_workers(&self, bg: &BackgroundRunner) { + pub fn spawn_workers(self: &Arc, bg: &BackgroundRunner) -> Result<(), Error> { self.block_manager.spawn_workers(bg); self.bucket_table.spawn_workers(bg); @@ -296,20 +284,49 @@ impl Garage { self.object_table.spawn_workers(bg); self.object_counter_table.spawn_workers(bg); + self.mpu_table.spawn_workers(bg); + self.mpu_counter_table.spawn_workers(bg); self.version_table.spawn_workers(bg); self.block_ref_table.spawn_workers(bg); + bg.spawn_worker(lifecycle_worker::LifecycleWorker::new( + self.clone(), + self.lifecycle_persister.clone(), + )); + #[cfg(feature = "k2v")] self.k2v.spawn_workers(bg); + + if let Some(itv) = self.config.metadata_auto_snapshot_interval.as_deref() { + let interval = parse_duration::parse(itv) + .ok_or_message("Invalid `metadata_auto_snapshot_interval`")?; + if interval < std::time::Duration::from_secs(600) { + return Err(Error::Message( + "metadata_auto_snapshot_interval too small or negative".into(), + )); + } + + bg.spawn_worker(crate::snapshot::AutoSnapshotWorker::new( + self.clone(), + interval, + )); + } + + Ok(()) } - pub fn bucket_helper(&self) -> helper::bucket::BucketHelper { + pub fn bucket_helper(&self) -> helper::bucket::BucketHelper<'_> { helper::bucket::BucketHelper(self) } - pub fn key_helper(&self) -> helper::key::KeyHelper { + pub fn key_helper(&self) -> helper::key::KeyHelper<'_> { helper::key::KeyHelper(self) } + + pub async fn locked_helper(&self) -> helper::locked::LockedHelper<'_> { + let lock = self.bucket_lock.lock().await; + helper::locked::LockedHelper(self, Some(lock)) + } } #[cfg(feature = "k2v")] diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 4a488d7f..e5506d7e 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -1,19 +1,15 @@ use std::time::Duration; -use garage_util::crdt::*; use garage_util::data::*; -use garage_util::error::{Error as GarageError, OkOrMessage}; +use garage_util::error::OkOrMessage; use garage_util::time::*; use garage_table::util::*; -use crate::bucket_alias_table::*; use crate::bucket_table::*; use crate::garage::Garage; use crate::helper::error::*; -use crate::helper::key::KeyHelper; use crate::key_table::*; -use crate::permission::BucketKeyPerm; use crate::s3::object_table::*; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -71,6 +67,49 @@ impl<'a> BucketHelper<'a> { } } + /// Find a bucket by its global alias or a prefix of its uuid + pub async fn admin_get_existing_matching_bucket( + &self, + pattern: &String, + ) -> Result { + if let Some(uuid) = self.resolve_global_bucket_name(pattern).await? { + return Ok(uuid); + } else if pattern.len() >= 2 { + let hexdec = pattern + .get(..pattern.len() & !1) + .and_then(|x| hex::decode(x).ok()); + if let Some(hex) = hexdec { + let mut start = [0u8; 32]; + start + .as_mut_slice() + .get_mut(..hex.len()) + .ok_or_bad_request("invalid length")? + .copy_from_slice(&hex); + let mut candidates = self + .0 + .bucket_table + .get_range( + &EmptyKey, + Some(start.into()), + Some(DeletedFilter::NotDeleted), + 10, + EnumerationOrder::Forward, + ) + .await? + .into_iter() + .collect::>(); + candidates.retain(|x| hex::encode(x.id).starts_with(pattern)); + if candidates.len() == 1 { + return Ok(candidates.into_iter().next().unwrap().id); + } + } + } + Err(Error::BadRequest(format!( + "Bucket not found / several matching buckets: {}", + pattern + ))) + } + /// Returns a Bucket if it is present in bucket table, /// even if it is in deleted state. Querying a non-existing /// bucket ID returns an internal error. @@ -96,341 +135,7 @@ impl<'a> BucketHelper<'a> { .ok_or_else(|| Error::NoSuchBucket(hex::encode(bucket_id))) } - /// Sets a new alias for a bucket in global namespace. - /// This function fails if: - /// - alias name is not valid according to S3 spec - /// - bucket does not exist or is deleted - /// - alias already exists and points to another bucket - pub async fn set_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - if !is_valid_bucket_name(alias_name) { - return Err(Error::InvalidBucketName(alias_name.to_string())); - } - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - - let alias = self.0.bucket_alias_table.get(&EmptyKey, alias_name).await?; - - if let Some(existing_alias) = alias.as_ref() { - if let Some(p_bucket) = existing_alias.state.get() { - if *p_bucket != bucket_id { - return Err(Error::BadRequest(format!( - "Alias {} already exists and points to different bucket: {:?}", - alias_name, p_bucket - ))); - } - } - } - - // Checks ok, add alias - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - - let alias_ts = increment_logical_clock_2( - bucket_p.aliases.get_timestamp(alias_name), - alias.as_ref().map(|a| a.state.timestamp()).unwrap_or(0), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - let alias = match alias { - None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)) - .ok_or_else(|| Error::InvalidBucketName(alias_name.clone()))?, - Some(mut a) => { - a.state = Lww::raw(alias_ts, Some(bucket_id)); - a - } - }; - self.0.bucket_alias_table.insert(&alias).await?; - - bucket_p.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, true); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Unsets an alias for a bucket in global namespace. - /// This function fails if: - /// - bucket does not exist or is deleted - /// - alias does not exist or maps to another bucket (-> internal error) - /// - bucket has no other aliases (global or local) - pub async fn unset_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut bucket_state = bucket.state.as_option_mut().unwrap(); - - let mut alias = self - .0 - .bucket_alias_table - .get(&EmptyKey, alias_name) - .await? - .filter(|a| a.state.get().map(|x| x == bucket_id).unwrap_or(false)) - .ok_or_message(format!( - "Internal error: alias not found or does not point to bucket {:?}", - bucket_id - ))?; - - let has_other_global_aliases = bucket_state - .aliases - .items() - .iter() - .any(|(name, _, active)| name != alias_name && *active); - let has_other_local_aliases = bucket_state - .local_aliases - .items() - .iter() - .any(|(_, _, active)| *active); - if !has_other_global_aliases && !has_other_local_aliases { - return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); - } - - // Checks ok, remove alias - let alias_ts = increment_logical_clock_2( - alias.state.timestamp(), - bucket_state.aliases.get_timestamp(alias_name), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - alias.state = Lww::raw(alias_ts, None); - self.0.bucket_alias_table.insert(&alias).await?; - - bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Ensures a bucket does not have a certain global alias. - /// Contrarily to unset_global_bucket_alias, this does not - /// fail on any condition other than: - /// - bucket cannot be found (its fine if it is in deleted state) - /// - alias cannot be found (its fine if it points to nothing or - /// to another bucket) - pub async fn purge_global_bucket_alias( - &self, - bucket_id: Uuid, - alias_name: &String, - ) -> Result<(), Error> { - let mut bucket = self.get_internal_bucket(bucket_id).await?; - - let mut alias = self - .0 - .bucket_alias_table - .get(&EmptyKey, alias_name) - .await? - .ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?; - - // Checks ok, remove alias - let alias_ts = match bucket.state.as_option() { - Some(bucket_state) => increment_logical_clock_2( - alias.state.timestamp(), - bucket_state.aliases.get_timestamp(alias_name), - ), - None => increment_logical_clock(alias.state.timestamp()), - }; - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - if alias.state.get() == &Some(bucket_id) { - alias.state = Lww::raw(alias_ts, None); - self.0.bucket_alias_table.insert(&alias).await?; - } - - if let Some(mut bucket_state) = bucket.state.as_option_mut() { - bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - } - - Ok(()) - } - - /// Sets a new alias for a bucket in the local namespace of a key. - /// This function fails if: - /// - alias name is not valid according to S3 spec - /// - bucket does not exist or is deleted - /// - key does not exist or is deleted - /// - alias already exists and points to another bucket - pub async fn set_local_bucket_alias( - &self, - bucket_id: Uuid, - key_id: &String, - alias_name: &String, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - if !is_valid_bucket_name(alias_name) { - return Err(Error::InvalidBucketName(alias_name.to_string())); - } - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = key_helper.get_existing_key(key_id).await?; - - let mut key_param = key.state.as_option_mut().unwrap(); - - if let Some(Some(existing_alias)) = key_param.local_aliases.get(alias_name) { - if *existing_alias != bucket_id { - return Err(Error::BadRequest(format!("Alias {} already exists in namespace of key {} and points to different bucket: {:?}", alias_name, key.key_id, existing_alias))); - } - } - - // Checks ok, add alias - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); - - // Calculate the timestamp to assign to this aliasing in the two local_aliases maps - // (the one from key to bucket, and the reverse one stored in the bucket iself) - // so that merges on both maps in case of a concurrent operation resolve - // to the same alias being set - let alias_ts = increment_logical_clock_2( - key_param.local_aliases.get_timestamp(alias_name), - bucket_p - .local_aliases - .get_timestamp(&bucket_p_local_alias_key), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id)); - self.0.key_table.insert(&key).await?; - - bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, true); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Unsets an alias for a bucket in the local namespace of a key. - /// This function fails if: - /// - bucket does not exist or is deleted - /// - key does not exist or is deleted - /// - alias does not exist or maps to another bucket (-> internal error) - /// - bucket has no other aliases (global or local) - pub async fn unset_local_bucket_alias( - &self, - bucket_id: Uuid, - key_id: &String, - alias_name: &String, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = key_helper.get_existing_key(key_id).await?; - - let mut bucket_p = bucket.state.as_option_mut().unwrap(); - - if key - .state - .as_option() - .unwrap() - .local_aliases - .get(alias_name) - .cloned() - .flatten() != Some(bucket_id) - { - return Err(GarageError::Message(format!( - "Bucket {:?} does not have alias {} in namespace of key {}", - bucket_id, alias_name, key_id - )) - .into()); - } - - let has_other_global_aliases = bucket_p - .aliases - .items() - .iter() - .any(|(_, _, active)| *active); - let has_other_local_aliases = bucket_p - .local_aliases - .items() - .iter() - .any(|((k, n), _, active)| *k == key.key_id && n == alias_name && *active); - if !has_other_global_aliases && !has_other_local_aliases { - return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); - } - - // Checks ok, remove alias - let mut key_param = key.state.as_option_mut().unwrap(); - let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); - - let alias_ts = increment_logical_clock_2( - key_param.local_aliases.get_timestamp(alias_name), - bucket_p - .local_aliases - .get_timestamp(&bucket_p_local_alias_key), - ); - - // ---- timestamp-ensured causality barrier ---- - // writes are now done and all writes use timestamp alias_ts - - key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None); - self.0.key_table.insert(&key).await?; - - bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); - self.0.bucket_table.insert(&bucket).await?; - - Ok(()) - } - - /// Sets permissions for a key on a bucket. - /// This function fails if: - /// - bucket or key cannot be found at all (its ok if they are in deleted state) - /// - bucket or key is in deleted state and we are trying to set permissions other than "deny - /// all" - pub async fn set_bucket_key_permissions( - &self, - bucket_id: Uuid, - key_id: &String, - mut perm: BucketKeyPerm, - ) -> Result<(), Error> { - let key_helper = KeyHelper(self.0); - - let mut bucket = self.get_internal_bucket(bucket_id).await?; - let mut key = key_helper.get_internal_key(key_id).await?; - - if let Some(bstate) = bucket.state.as_option() { - if let Some(kp) = bstate.authorized_keys.get(key_id) { - perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp); - } - } else if perm.is_any() { - return Err(Error::BadRequest( - "Trying to give permissions on a deleted bucket".into(), - )); - } - - if let Some(kstate) = key.state.as_option() { - if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) { - perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp); - } - } else if perm.is_any() { - return Err(Error::BadRequest( - "Trying to give permissions to a deleted key".into(), - )); - } - - // ---- timestamp-ensured causality barrier ---- - - if let Some(bstate) = bucket.state.as_option_mut() { - bstate.authorized_keys = Map::put_mutator(key_id.clone(), perm); - self.0.bucket_table.insert(&bucket).await?; - } - - if let Some(kstate) = key.state.as_option_mut() { - kstate.authorized_buckets = Map::put_mutator(bucket_id, perm); - self.0.key_table.insert(&key).await?; - } - - Ok(()) - } + // ---- pub async fn is_bucket_empty(&self, bucket_id: Uuid) -> Result { let objects = self @@ -450,10 +155,12 @@ impl<'a> BucketHelper<'a> { #[cfg(feature = "k2v")] { - use garage_rpc::ring::Ring; - use std::sync::Arc; - - let ring: Arc = self.0.system.ring.borrow().clone(); + let node_id_vec = self + .0 + .system + .cluster_layout() + .all_nongateway_nodes() + .to_vec(); let k2vindexes = self .0 .k2v @@ -462,7 +169,7 @@ impl<'a> BucketHelper<'a> { .get_range( &bucket_id, None, - Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())), + Some((DeletedFilter::NotDeleted, node_id_vec)), 10, EnumerationOrder::Forward, ) @@ -478,7 +185,9 @@ impl<'a> BucketHelper<'a> { // ---- /// Deletes all incomplete multipart uploads that are older than a certain time. - /// Returns the number of uploads aborted + /// Returns the number of uploads aborted. + /// This will also include non-multipart uploads, which may be lingering + /// after a node crash pub async fn cleanup_incomplete_uploads( &self, bucket_id: &Uuid, @@ -496,7 +205,9 @@ impl<'a> BucketHelper<'a> { .get_range( bucket_id, start, - Some(ObjectFilter::IsUploading), + Some(ObjectFilter::IsUploading { + check_multipart: None, + }), 1000, EnumerationOrder::Forward, ) @@ -508,7 +219,7 @@ impl<'a> BucketHelper<'a> { let aborted_versions = object .versions() .iter() - .filter(|v| v.is_uploading() && v.timestamp < older_than) + .filter(|v| v.is_uploading(None) && v.timestamp < older_than) .map(|v| ObjectVersion { state: ObjectVersionState::Aborted, uuid: v.uuid, diff --git a/src/model/helper/error.rs b/src/model/helper/error.rs index 3ca8f55c..bc483c7d 100644 --- a/src/model/helper/error.rs +++ b/src/model/helper/error.rs @@ -1,30 +1,30 @@ -use err_derive::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; use garage_util::error::Error as GarageError; #[derive(Debug, Error, Serialize, Deserialize)] pub enum Error { - #[error(display = "Internal error: {}", _0)] - Internal(#[error(source)] GarageError), + #[error("Internal error: {0}")] + Internal(#[from] GarageError), - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), } -impl From for Error { - fn from(e: netapp::error::Error) -> Self { - Error::Internal(GarageError::Netapp(e)) +impl From for Error { + fn from(e: garage_net::error::Error) -> Self { + Error::Internal(GarageError::Net(e)) } } diff --git a/src/model/helper/key.rs b/src/model/helper/key.rs index c1a8e974..b8a99d55 100644 --- a/src/model/helper/key.rs +++ b/src/model/helper/key.rs @@ -1,12 +1,9 @@ use garage_table::util::*; -use garage_util::crdt::*; use garage_util::error::OkOrMessage; use crate::garage::Garage; -use crate::helper::bucket::BucketHelper; use crate::helper::error::*; use crate::key_table::{Key, KeyFilter}; -use crate::permission::BucketKeyPerm; pub struct KeyHelper<'a>(pub(crate) &'a Garage); @@ -65,38 +62,4 @@ impl<'a> KeyHelper<'a> { Ok(candidates.into_iter().next().unwrap()) } } - - /// Deletes an API access key - pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> { - let bucket_helper = BucketHelper(self.0); - - let state = key.state.as_option_mut().unwrap(); - - // --- done checking, now commit --- - // (the step at unset_local_bucket_alias will fail if a bucket - // does not have another alias, the deletion will be - // interrupted in the middle if that happens) - - // 1. Delete local aliases - for (alias, _, to) in state.local_aliases.items().iter() { - if let Some(bucket_id) = to { - bucket_helper - .unset_local_bucket_alias(*bucket_id, &key.key_id, alias) - .await?; - } - } - - // 2. Remove permissions on all authorized buckets - for (ab_id, _auth) in state.authorized_buckets.items().iter() { - bucket_helper - .set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) - .await?; - } - - // 3. Actually delete key - key.state = Deletable::delete(); - self.0.key_table.insert(key).await?; - - Ok(()) - } } diff --git a/src/model/helper/locked.rs b/src/model/helper/locked.rs new file mode 100644 index 00000000..98344b63 --- /dev/null +++ b/src/model/helper/locked.rs @@ -0,0 +1,654 @@ +use std::collections::{HashMap, HashSet}; + +use garage_db as db; + +use garage_util::crdt::*; +use garage_util::data::*; +use garage_util::error::{Error as GarageError, OkOrMessage}; +use garage_util::time::*; + +use garage_table::util::*; + +use crate::bucket_alias_table::*; +use crate::garage::Garage; +use crate::helper::bucket::BucketHelper; +use crate::helper::error::*; +use crate::helper::key::KeyHelper; +use crate::key_table::*; +use crate::permission::BucketKeyPerm; + +/// A LockedHelper is the mandatory struct to hold when doing operations +/// that modify access keys or bucket aliases. This structure takes +/// a lock to a unit value that is in the globally-shared Garage struct. +/// +/// This avoid several concurrent requests to modify the list of buckets +/// and aliases at the same time, ending up in inconsistent states. +/// This DOES NOT FIX THE FUNDAMENTAL ISSUE as CreateBucket requests handled +/// by different API nodes can still break the cluster, but it is a first +/// fix that allows consistency to be maintained if all such requests are +/// directed to a single node, which is doable for many deployments. +/// +/// See issues: #649, #723 +pub struct LockedHelper<'a>( + pub(crate) &'a Garage, + pub(crate) Option>, +); + +impl<'a> Drop for LockedHelper<'a> { + fn drop(&mut self) { + // make it explicit that the mutexguard lives until here + drop(self.1.take()) + } +} + +#[allow(clippy::ptr_arg)] +impl<'a> LockedHelper<'a> { + pub fn bucket(&self) -> BucketHelper<'a> { + BucketHelper(self.0) + } + + pub fn key(&self) -> KeyHelper<'a> { + KeyHelper(self.0) + } + + // ================================================ + // global bucket aliases + // ================================================ + + /// Sets a new alias for a bucket in global namespace. + /// This function fails if: + /// - alias name is not valid according to S3 spec + /// - bucket does not exist or is deleted + /// - alias already exists and points to another bucket + pub async fn set_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) { + return Err(Error::InvalidBucketName(alias_name.to_string())); + } + + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + + let alias = self.0.bucket_alias_table.get(&EmptyKey, alias_name).await?; + + if let Some(existing_alias) = alias.as_ref() { + if let Some(p_bucket) = existing_alias.state.get() { + if *p_bucket != bucket_id { + return Err(Error::BadRequest(format!( + "Alias {} already exists and points to different bucket: {:?}", + alias_name, p_bucket + ))); + } + } + } + + // Checks ok, add alias + let bucket_p = bucket.state.as_option_mut().unwrap(); + + let alias_ts = increment_logical_clock_2( + bucket_p.aliases.get_timestamp(alias_name), + alias.as_ref().map(|a| a.state.timestamp()).unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + let alias = match alias { + None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)), + Some(mut a) => { + a.state = Lww::raw(alias_ts, Some(bucket_id)); + a + } + }; + self.0.bucket_alias_table.insert(&alias).await?; + + bucket_p.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, true); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Unsets an alias for a bucket in global namespace. + /// This function fails if: + /// - bucket does not exist or is deleted + /// - alias does not exist or maps to another bucket (-> internal error) + /// - bucket has no other aliases (global or local) + pub async fn unset_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let bucket_state = bucket.state.as_option_mut().unwrap(); + + let mut alias = self + .0 + .bucket_alias_table + .get(&EmptyKey, alias_name) + .await? + .filter(|a| a.state.get().map(|x| x == bucket_id).unwrap_or(false)) + .ok_or_message(format!( + "Internal error: alias not found or does not point to bucket {:?}", + bucket_id + ))?; + + let has_other_global_aliases = bucket_state + .aliases + .items() + .iter() + .any(|(name, _, active)| name != alias_name && *active); + let has_other_local_aliases = bucket_state + .local_aliases + .items() + .iter() + .any(|(_, _, active)| *active); + if !has_other_global_aliases && !has_other_local_aliases { + return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); + } + + // Checks ok, remove alias + let alias_ts = increment_logical_clock_2( + alias.state.timestamp(), + bucket_state.aliases.get_timestamp(alias_name), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + alias.state = Lww::raw(alias_ts, None); + self.0.bucket_alias_table.insert(&alias).await?; + + bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Ensures a bucket does not have a certain global alias. + /// Contrarily to unset_global_bucket_alias, this does not + /// fail on any condition other than: + /// - bucket cannot be found (its fine if it is in deleted state) + /// - alias cannot be found (its fine if it points to nothing or + /// to another bucket) + pub async fn purge_global_bucket_alias( + &self, + bucket_id: Uuid, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + + let mut alias = self + .0 + .bucket_alias_table + .get(&EmptyKey, alias_name) + .await? + .ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?; + + // Checks ok, remove alias + let alias_ts = increment_logical_clock_2( + alias.state.timestamp(), + bucket + .state + .as_option() + .map(|p| p.aliases.get_timestamp(alias_name)) + .unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + if alias.state.get() == &Some(bucket_id) { + alias.state = Lww::raw(alias_ts, None); + self.0.bucket_alias_table.insert(&alias).await?; + } + + if let Some(bucket_state) = bucket.state.as_option_mut() { + bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + } + + Ok(()) + } + + // ================================================ + // local bucket aliases + // ================================================ + + /// Sets a new alias for a bucket in the local namespace of a key. + /// This function fails if: + /// - alias name is not valid according to S3 spec + /// - bucket does not exist or is deleted + /// - key does not exist or is deleted + /// - alias already exists and points to another bucket + pub async fn set_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) { + return Err(Error::InvalidBucketName(alias_name.to_string())); + } + + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let mut key = self.key().get_existing_key(key_id).await?; + + let key_param = key.state.as_option_mut().unwrap(); + + if let Some(Some(existing_alias)) = key_param.local_aliases.get(alias_name) { + if *existing_alias != bucket_id { + return Err(Error::BadRequest(format!("Alias {} already exists in namespace of key {} and points to different bucket: {:?}", alias_name, key.key_id, existing_alias))); + } + } + + // Checks ok, add alias + let bucket_p = bucket.state.as_option_mut().unwrap(); + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + // Calculate the timestamp to assign to this aliasing in the two local_aliases maps + // (the one from key to bucket, and the reverse one stored in the bucket itself) + // so that merges on both maps in case of a concurrent operation resolve + // to the same alias being set + let alias_ts = increment_logical_clock_2( + key_param.local_aliases.get_timestamp(alias_name), + bucket_p + .local_aliases + .get_timestamp(&bucket_p_local_alias_key), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id)); + self.0.key_table.insert(&key).await?; + + bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, true); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Unsets an alias for a bucket in the local namespace of a key. + /// This function fails if: + /// - bucket does not exist or is deleted + /// - key does not exist or is deleted + /// - alias does not exist or maps to another bucket (-> internal error) + /// - bucket has no other aliases (global or local) + pub async fn unset_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?; + let mut key = self.key().get_existing_key(key_id).await?; + + let key_p = key.state.as_option().unwrap(); + let bucket_p = bucket.state.as_option_mut().unwrap(); + + if key_p.local_aliases.get(alias_name).cloned().flatten() != Some(bucket_id) { + return Err(GarageError::Message(format!( + "Bucket {:?} does not have alias {} in namespace of key {}", + bucket_id, alias_name, key_id + )) + .into()); + } + + let has_other_global_aliases = bucket_p + .aliases + .items() + .iter() + .any(|(_, _, active)| *active); + let has_other_local_aliases = bucket_p + .local_aliases + .items() + .iter() + .any(|((k, n), _, active)| (*k != key.key_id || n != alias_name) && *active); + + if !has_other_global_aliases && !has_other_local_aliases { + return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name))); + } + + // Checks ok, remove alias + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + let alias_ts = increment_logical_clock_2( + key_p.local_aliases.get_timestamp(alias_name), + bucket_p + .local_aliases + .get_timestamp(&bucket_p_local_alias_key), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + key.state.as_option_mut().unwrap().local_aliases = + LwwMap::raw_item(alias_name.clone(), alias_ts, None); + self.0.key_table.insert(&key).await?; + + bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + + Ok(()) + } + + /// Ensures a bucket does not have a certain local alias. + /// Contrarily to unset_local_bucket_alias, this does not + /// fail on any condition other than: + /// - bucket cannot be found (its fine if it is in deleted state) + /// - key cannot be found (its fine if alias in key points to nothing + /// or to another bucket) + pub async fn purge_local_bucket_alias( + &self, + bucket_id: Uuid, + key_id: &String, + alias_name: &String, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + let mut key = self.key().get_internal_key(key_id).await?; + + let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone()); + + let alias_ts = increment_logical_clock_2( + key.state + .as_option() + .map(|p| p.local_aliases.get_timestamp(alias_name)) + .unwrap_or(0), + bucket + .state + .as_option() + .map(|p| p.local_aliases.get_timestamp(&bucket_p_local_alias_key)) + .unwrap_or(0), + ); + + // ---- timestamp-ensured causality barrier ---- + // writes are now done and all writes use timestamp alias_ts + + if let Some(kp) = key.state.as_option_mut() { + kp.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None); + self.0.key_table.insert(&key).await?; + } + + if let Some(bp) = bucket.state.as_option_mut() { + bp.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false); + self.0.bucket_table.insert(&bucket).await?; + } + + Ok(()) + } + + // ================================================ + // permissions + // ================================================ + + /// Sets permissions for a key on a bucket. + /// This function fails if: + /// - bucket or key cannot be found at all (its ok if they are in deleted state) + /// - bucket or key is in deleted state and we are trying to set + /// permissions other than "deny all" + pub async fn set_bucket_key_permissions( + &self, + bucket_id: Uuid, + key_id: &String, + mut perm: BucketKeyPerm, + ) -> Result<(), Error> { + let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?; + let mut key = self.key().get_internal_key(key_id).await?; + + if let Some(bstate) = bucket.state.as_option() { + if let Some(kp) = bstate.authorized_keys.get(key_id) { + perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp); + } + } else if perm.is_any() { + return Err(Error::BadRequest( + "Trying to give permissions on a deleted bucket".into(), + )); + } + + if let Some(kstate) = key.state.as_option() { + if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) { + perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp); + } + } else if perm.is_any() { + return Err(Error::BadRequest( + "Trying to give permissions to a deleted key".into(), + )); + } + + // ---- timestamp-ensured causality barrier ---- + + if let Some(bstate) = bucket.state.as_option_mut() { + bstate.authorized_keys = Map::put_mutator(key_id.clone(), perm); + self.0.bucket_table.insert(&bucket).await?; + } + + if let Some(kstate) = key.state.as_option_mut() { + kstate.authorized_buckets = Map::put_mutator(bucket_id, perm); + self.0.key_table.insert(&key).await?; + } + + Ok(()) + } + + // ================================================ + // keys + // ================================================ + + /// Deletes an API access key + pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> { + let state = key.state.as_option_mut().unwrap(); + + // --- done checking, now commit --- + + // 1. Delete local aliases + for (alias, _, to) in state.local_aliases.items().iter() { + if let Some(bucket_id) = to { + self.purge_local_bucket_alias(*bucket_id, &key.key_id, alias) + .await?; + } + } + + // 2. Remove permissions on all authorized buckets + for (ab_id, _auth) in state.authorized_buckets.items().iter() { + self.set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) + .await?; + } + + // 3. Actually delete key + key.state = Deletable::delete(); + self.0.key_table.insert(key).await?; + + Ok(()) + } + + // ================================================ + // repair procedure + // ================================================ + + pub async fn repair_aliases(&self) -> Result<(), GarageError> { + self.0.db.transaction(|tx| { + info!("--- begin repair_aliases transaction ----"); + + // 1. List all non-deleted buckets, so that we can fix bad aliases + let mut all_buckets: HashSet = HashSet::new(); + + for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? { + let bucket = self + .0 + .bucket_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + if !bucket.is_deleted() { + all_buckets.insert(bucket.id); + } + } + + info!("number of buckets: {}", all_buckets.len()); + + // 2. List all aliases declared in bucket_alias_table and key_table + // Take note of aliases that point to non-existing buckets + let mut global_aliases: HashMap = HashMap::new(); + + { + let mut delete_global = vec![]; + for item in tx.range::<&[u8], _>(&self.0.bucket_alias_table.data.store, ..)? { + let mut alias = self + .0 + .bucket_alias_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + if let Some(id) = alias.state.get() { + if all_buckets.contains(id) { + // keep aliases + global_aliases.insert(alias.name().to_string(), *id); + } else { + // delete alias + warn!( + "global alias: remove {} -> {:?} (bucket is deleted)", + alias.name(), + id + ); + alias.state.update(None); + delete_global.push(alias); + } + } + } + + info!("number of global aliases: {}", global_aliases.len()); + + info!("global alias table: {} entries fixed", delete_global.len()); + for ga in delete_global { + debug!("Enqueue update to global alias table: {:?}", ga); + self.0.bucket_alias_table.queue_insert(tx, &ga)?; + } + } + + let mut local_aliases: HashMap<(String, String), Uuid> = HashMap::new(); + + { + let mut delete_local = vec![]; + + for item in tx.range::<&[u8], _>(&self.0.key_table.data.store, ..)? { + let mut key = self + .0 + .key_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + let Some(p) = key.state.as_option_mut() else { + continue; + }; + let mut has_changes = false; + for (name, _, to) in p.local_aliases.items().to_vec() { + if let Some(id) = to { + if all_buckets.contains(&id) { + local_aliases.insert((key.key_id.clone(), name), id); + } else { + warn!( + "local alias: remove ({}, {}) -> {:?} (bucket is deleted)", + key.key_id, name, id + ); + p.local_aliases.update_in_place(name, None); + has_changes = true; + } + } + } + if has_changes { + delete_local.push(key); + } + } + + info!("number of local aliases: {}", local_aliases.len()); + + info!("key table: {} entries fixed", delete_local.len()); + for la in delete_local { + debug!("Enqueue update to key table: {:?}", la); + self.0.key_table.queue_insert(tx, &la)?; + } + } + + // 4. Reverse the alias maps to determine the aliases per-bucket + let mut bucket_global: HashMap> = HashMap::new(); + let mut bucket_local: HashMap> = HashMap::new(); + + for (name, bucket) in global_aliases { + bucket_global.entry(bucket).or_default().push(name); + } + for ((key, name), bucket) in local_aliases { + bucket_local.entry(bucket).or_default().push((key, name)); + } + + // 5. Fix the bucket table to ensure consistency + let mut bucket_updates = vec![]; + + for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? { + let bucket = self + .0 + .bucket_table + .data + .decode_entry(&(item?.1)) + .map_err(db::TxError::Abort)?; + let mut bucket2 = bucket.clone(); + let Some(param) = bucket2.state.as_option_mut() else { + continue; + }; + + // fix global aliases + { + let ga = bucket_global.remove(&bucket.id).unwrap_or_default(); + for (name, _, active) in param.aliases.items().to_vec() { + if active && !ga.contains(&name) { + warn!("bucket {:?}: remove global alias {}", bucket.id, name); + param.aliases.update_in_place(name, false); + } + } + for name in ga { + if param.aliases.get(&name).copied() != Some(true) { + warn!("bucket {:?}: add global alias {}", bucket.id, name); + param.aliases.update_in_place(name, true); + } + } + } + + // fix local aliases + { + let la = bucket_local.remove(&bucket.id).unwrap_or_default(); + for (pair, _, active) in param.local_aliases.items().to_vec() { + if active && !la.contains(&pair) { + warn!("bucket {:?}: remove local alias {:?}", bucket.id, pair); + param.local_aliases.update_in_place(pair, false); + } + } + for pair in la { + if param.local_aliases.get(&pair).copied() != Some(true) { + warn!("bucket {:?}: add local alias {:?}", bucket.id, pair); + param.local_aliases.update_in_place(pair, true); + } + } + } + + if bucket2 != bucket { + bucket_updates.push(bucket2); + } + } + + info!("bucket table: {} entries fixed", bucket_updates.len()); + for b in bucket_updates { + debug!("Enqueue update to bucket table: {:?}", b); + self.0.bucket_table.queue_insert(tx, &b)?; + } + + info!("--- end repair_aliases transaction ----"); + + Ok(()) + })?; + + info!("repair_aliases is done"); + + Ok(()) + } +} diff --git a/src/model/helper/mod.rs b/src/model/helper/mod.rs index dd947c86..5534ae3d 100644 --- a/src/model/helper/mod.rs +++ b/src/model/helper/mod.rs @@ -1,3 +1,4 @@ pub mod bucket; pub mod error; pub mod key; +pub mod locked; diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 35d6596d..aa13ee7b 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use garage_db as db; -use garage_rpc::ring::Ring; +use garage_rpc::layout::LayoutHelper; use garage_rpc::system::System; use garage_util::background::BackgroundRunner; use garage_util::data::*; @@ -83,9 +83,9 @@ impl Entry for CounterEntry { } impl CounterEntry { - pub fn filtered_values(&self, ring: &Ring) -> HashMap { - let nodes = &ring.layout.node_id_vec[..]; - self.filtered_values_with_nodes(nodes) + pub fn filtered_values(&self, layout: &LayoutHelper) -> HashMap { + let nodes = layout.all_nongateway_nodes(); + self.filtered_values_with_nodes(&nodes) } pub fn filtered_values_with_nodes(&self, nodes: &[Uuid]) -> HashMap { @@ -232,7 +232,7 @@ impl IndexCounter { let now = now_msec(); for (s, inc) in counts.iter() { - let mut ent = entry.values.entry(s.to_string()).or_insert((0, 0)); + let ent = entry.values.entry(s.to_string()).or_insert((0, 0)); ent.0 = std::cmp::max(ent.0 + 1, now); ent.1 += *inc; } @@ -294,7 +294,7 @@ impl IndexCounter { let counter_entry = local_counter.into_counter_entry(self.this_node); self.local_counter .db() - .transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?; + .transaction(|tx| self.table.queue_insert(tx, &counter_entry))?; next_start = Some(local_counter_k); } @@ -348,7 +348,7 @@ impl IndexCounter { }, }; for (s, v) in counts.iter() { - let mut tv = local_counter.values.entry(s.to_string()).or_insert((0, 0)); + let tv = local_counter.values.entry(s.to_string()).or_insert((0, 0)); tv.0 = std::cmp::max(tv.0 + 1, now); tv.1 += v; } @@ -360,7 +360,7 @@ impl IndexCounter { let counter_entry = local_counter.into_counter_entry(self.this_node); self.local_counter .db() - .transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?; + .transaction(|tx| self.table.queue_insert(tx, &counter_entry))?; next_start = Some(counted_entry_k); } diff --git a/src/model/k2v/causality.rs b/src/model/k2v/causality.rs index c80ebd39..7d311ede 100644 --- a/src/model/k2v/causality.rs +++ b/src/model/k2v/causality.rs @@ -16,8 +16,6 @@ use serde::{Deserialize, Serialize}; use garage_util::data::*; -use crate::helper::error::{Error as HelperError, OkOrBadRequest}; - /// Node IDs used in K2V are u64 integers that are the abbreviation /// of full Garage node IDs which are 256-bit UUIDs. pub type K2VNodeId = u64; @@ -99,10 +97,6 @@ impl CausalContext { Some(ret) } - pub fn parse_helper(s: &str) -> Result { - Self::parse(s).ok_or_bad_request("Invalid causality token") - } - /// Check if this causal context contains newer items than another one pub fn is_newer_than(&self, other: &Self) -> bool { vclock_gt(&self.vector_clock, &other.vector_clock) diff --git a/src/model/k2v/rpc.rs b/src/model/k2v/rpc.rs index 37e142f6..821f4549 100644 --- a/src/model/k2v/rpc.rs +++ b/src/model/k2v/rpc.rs @@ -10,7 +10,6 @@ use std::convert::TryInto; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; -use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::StreamExt; use serde::{Deserialize, Serialize}; @@ -127,23 +126,21 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&partition.hash()); + .storage_nodes(&partition.hash()); who.sort(); self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, K2VRpc::InsertItem(InsertedItem { partition, sort_key, causal_context, value, }), - RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(1) - .interrupt_after_quorum(true), + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(1), ) .await?; @@ -168,7 +165,7 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&partition.hash()); + .storage_nodes(&partition.hash()); who.sort(); call_list.entry(who).or_default().push(InsertedItem { @@ -187,14 +184,12 @@ impl K2VRpcHandler { let call_futures = call_list.into_iter().map(|(nodes, items)| async move { let resp = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, &nodes[..], K2VRpc::InsertManyItems(items), - RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(1) - .interrupt_after_quorum(true), + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(1), ) .await?; Ok::<_, Error>((nodes, resp)) @@ -227,11 +222,11 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&poll_key.partition.hash()); + .storage_nodes(&poll_key.partition.hash()); - let rpc = self.system.rpc.try_call_many( + let rpc = self.system.rpc_helper().try_call_many( &self.endpoint, - &nodes[..], + &nodes, K2VRpc::PollItem { key: poll_key, causal_context, @@ -239,9 +234,10 @@ impl K2VRpcHandler { }, RequestStrategy::with_priority(PRIO_NORMAL) .with_quorum(self.item_table.data.replication.read_quorum()) + .send_all_at_once(true) .without_timeout(), ); - let timeout_duration = Duration::from_millis(timeout_msec) + self.system.rpc.rpc_timeout(); + let timeout_duration = Duration::from_millis(timeout_msec); let resps = select! { r = rpc => r?, _ = tokio::time::sleep(timeout_duration) => return Ok(None), @@ -287,7 +283,7 @@ impl K2VRpcHandler { .item_table .data .replication - .write_nodes(&range.partition.hash()); + .storage_nodes(&range.partition.hash()); let quorum = self.item_table.data.replication.read_quorum(); let msg = K2VRpc::PollRange { range, @@ -296,11 +292,15 @@ impl K2VRpcHandler { }; // Send the request to all nodes, use FuturesUnordered to get the responses in any order - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; let rs = RequestStrategy::with_priority(PRIO_NORMAL).without_timeout(); let mut requests = nodes .iter() - .map(|node| self.system.rpc.call(&self.endpoint, *node, msg.clone(), rs)) + .map(|node| { + self.system + .rpc_helper() + .call(&self.endpoint, *node, msg.clone(), rs.clone()) + }) .collect::>(); // Fetch responses. This procedure stops fetching responses when any of the following @@ -309,15 +309,14 @@ impl K2VRpcHandler { // - we have a response to a read quorum of requests (e.g. 2/3), and an extra delay // has passed since the quorum was achieved // - a global RPC timeout expired - // The extra delay after a quorum was received is usefull if the third response was to + // The extra delay after a quorum was received is useful if the third response was to // arrive during this short interval: this would allow us to consider all the data seen // by that last node in the response we produce, and would likely help reduce the // size of the seen marker that we will return (because we would have an info of the // kind: all items produced by that node until time ts have been returned, so we can // bump the entry in the global vector clock and possibly remove some item-specific // vector clocks) - let mut deadline = - Instant::now() + Duration::from_millis(timeout_msec) + self.system.rpc.rpc_timeout(); + let mut deadline = Instant::now() + Duration::from_millis(timeout_msec); let mut resps = vec![]; let mut errors = vec![]; loop { @@ -339,7 +338,7 @@ impl K2VRpcHandler { } if errors.len() > nodes.len() - quorum { let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - return Err(Error::Quorum(quorum, resps.len(), nodes.len(), errors).into()); + return Err(Error::Quorum(quorum, None, resps.len(), nodes.len(), errors).into()); } // Take all returned items into account to produce the response. @@ -500,7 +499,7 @@ impl K2VRpcHandler { } else { // If no seen marker was specified, we do not poll for anything. // We return immediately with the set of known items (even if - // it is empty), which will give the client an inital view of + // it is empty), which will give the client an initial view of // the dataset and an initial seen marker for further // PollRange calls. self.poll_range_read_range(range, &RangeSeenMarker::default()) @@ -537,7 +536,6 @@ impl K2VRpcHandler { } } -#[async_trait] impl EndpointHandler for K2VRpcHandler { async fn handle(self: &Arc, message: &K2VRpc, _from: NodeID) -> Result { match message { diff --git a/src/model/k2v/seen.rs b/src/model/k2v/seen.rs index 8fe3a582..59d4ca5b 100644 --- a/src/model/k2v/seen.rs +++ b/src/model/k2v/seen.rs @@ -79,7 +79,7 @@ impl RangeSeenMarker { let bytes = nonversioned_encode(&self)?; let bytes = zstd::stream::encode_all(&mut &bytes[..], zstd::DEFAULT_COMPRESSION_LEVEL)?; - Ok(BASE64_STANDARD.encode(&bytes)) + Ok(BASE64_STANDARD.encode(bytes)) } /// Decode from msgpack+zstd+b64 representation, returns None on error. diff --git a/src/model/key_table.rs b/src/model/key_table.rs index bb5334a3..efb95f08 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -7,48 +7,7 @@ use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema}; use crate::permission::BucketKeyPerm; -pub(crate) mod v05 { - use garage_util::crdt; - use serde::{Deserialize, Serialize}; - - /// An api key - #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct Key { - /// The id of the key (immutable), used as partition key - pub key_id: String, - - /// The secret_key associated - pub secret_key: String, - - /// Name for the key - pub name: crdt::Lww, - - /// Is the key deleted - pub deleted: crdt::Bool, - - /// Buckets in which the key is authorized. Empty if `Key` is deleted - // CRDT interaction: deleted implies authorized_buckets is empty - pub authorized_buckets: crdt::LwwMap, - } - - /// Permission given to a key in a bucket - #[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct PermissionSet { - /// The key can be used to read the bucket - pub allow_read: bool, - /// The key can be used to write in the bucket - pub allow_write: bool, - } - - impl crdt::AutoCrdt for PermissionSet { - const WARN_IF_DIFFERENT: bool = true; - } - - impl garage_util::migrate::InitialFormat for Key {} -} - mod v08 { - use super::v05; use crate::permission::BucketKeyPerm; use garage_util::crdt; use garage_util::data::Uuid; @@ -86,32 +45,7 @@ mod v08 { pub local_aliases: crdt::LwwMap>, } - impl garage_util::migrate::Migrate for Key { - type Previous = v05::Key; - - fn migrate(old_k: v05::Key) -> Key { - let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone()); - - let state = if old_k.deleted.get() { - crdt::Deletable::Deleted - } else { - // Authorized buckets is ignored here, - // migration is performed in specific migration code in - // garage/migrate.rs - crdt::Deletable::Present(KeyParams { - secret_key: old_k.secret_key, - name, - allow_create_bucket: crdt::Lww::new(false), - authorized_buckets: crdt::Map::new(), - local_aliases: crdt::LwwMap::new(), - }) - }; - Key { - key_id: old_k.key_id, - state, - } - } - } + impl garage_util::migrate::InitialFormat for Key {} } pub use v08::*; @@ -149,11 +83,19 @@ impl Key { } /// Import a key from it's parts - pub fn import(key_id: &str, secret_key: &str, name: &str) -> Self { - Self { + pub fn import(key_id: &str, secret_key: &str, name: &str) -> Result { + if key_id.len() != 26 || &key_id[..2] != "GK" || hex::decode(&key_id[2..]).is_err() { + return Err("The specified key ID is not a valid Garage key ID (starts with `GK`, followed by 12 hex-encoded bytes)"); + } + + if secret_key.len() != 64 || hex::decode(&secret_key).is_err() { + return Err("The specified secret key is not a valid Garage secret key (composed of 32 hex-encoded bytes)"); + } + + Ok(Self { key_id: key_id.to_string(), state: crdt::Deletable::present(KeyParams::new(secret_key, name)), - } + }) } /// Create a new Key which can me merged to mark an existing key deleted diff --git a/src/model/lib.rs b/src/model/lib.rs index 4f20ea46..1939a7a9 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -1,9 +1,6 @@ #[macro_use] extern crate tracing; -// For migration from previous versions -pub(crate) mod prev; - pub mod permission; pub mod index_counter; @@ -18,4 +15,4 @@ pub mod s3; pub mod garage; pub mod helper; -pub mod migrate; +pub mod snapshot; diff --git a/src/model/migrate.rs b/src/model/migrate.rs deleted file mode 100644 index 6b4c3eed..00000000 --- a/src/model/migrate.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::sync::Arc; - -use garage_util::crdt::*; -use garage_util::data::*; -use garage_util::encode::nonversioned_decode; -use garage_util::error::Error as GarageError; -use garage_util::time::*; - -use crate::prev::v051::bucket_table as old_bucket; - -use crate::bucket_alias_table::*; -use crate::bucket_table::*; -use crate::garage::Garage; -use crate::helper::error::*; -use crate::permission::*; - -pub struct Migrate { - pub garage: Arc, -} - -impl Migrate { - pub async fn migrate_buckets050(&self) -> Result<(), Error> { - let tree = self - .garage - .db - .open_tree("bucket:table") - .map_err(GarageError::from)?; - - let mut old_buckets = vec![]; - for res in tree.iter().map_err(GarageError::from)? { - let (_k, v) = res.map_err(GarageError::from)?; - let bucket = - nonversioned_decode::(&v[..]).map_err(GarageError::from)?; - old_buckets.push(bucket); - } - - for bucket in old_buckets { - if let old_bucket::BucketState::Present(p) = bucket.state.get() { - self.migrate_buckets050_do_bucket(&bucket, p).await?; - } - } - - Ok(()) - } - - pub async fn migrate_buckets050_do_bucket( - &self, - old_bucket: &old_bucket::Bucket, - old_bucket_p: &old_bucket::BucketParams, - ) -> Result<(), Error> { - let bucket_id = blake2sum(old_bucket.name.as_bytes()); - - let new_name = if is_valid_bucket_name(&old_bucket.name) { - old_bucket.name.clone() - } else { - // if old bucket name was not valid, replace it by - // a hex-encoded name derived from its identifier - hex::encode(&bucket_id.as_slice()[..16]) - }; - - let website = if *old_bucket_p.website.get() { - Some(WebsiteConfig { - index_document: "index.html".into(), - error_document: None, - }) - } else { - None - }; - - self.garage - .bucket_table - .insert(&Bucket { - id: bucket_id, - state: Deletable::Present(BucketParams { - creation_date: now_msec(), - authorized_keys: Map::new(), - aliases: LwwMap::new(), - local_aliases: LwwMap::new(), - website_config: Lww::new(website), - cors_config: Lww::new(None), - quotas: Lww::new(Default::default()), - }), - }) - .await?; - - self.garage - .bucket_helper() - .set_global_bucket_alias(bucket_id, &new_name) - .await?; - - for (k, ts, perm) in old_bucket_p.authorized_keys.items().iter() { - self.garage - .bucket_helper() - .set_bucket_key_permissions( - bucket_id, - k, - BucketKeyPerm { - timestamp: *ts, - allow_read: perm.allow_read, - allow_write: perm.allow_write, - allow_owner: false, - }, - ) - .await?; - } - - Ok(()) - } -} diff --git a/src/model/prev/mod.rs b/src/model/prev/mod.rs deleted file mode 100644 index 68bb1502..00000000 --- a/src/model/prev/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod v051; diff --git a/src/model/prev/v051/bucket_table.rs b/src/model/prev/v051/bucket_table.rs deleted file mode 100644 index 19893458..00000000 --- a/src/model/prev/v051/bucket_table.rs +++ /dev/null @@ -1,63 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use garage_table::crdt::Crdt; -use garage_table::*; - -use crate::key_table::v05::PermissionSet; - -/// A bucket is a collection of objects -/// -/// Its parameters are not directly accessible as: -/// - It must be possible to merge paramaters, hence the use of a LWW CRDT. -/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub struct Bucket { - /// Name of the bucket - pub name: String, - /// State, and configuration if not deleted, of the bucket - pub state: crdt::Lww, -} - -/// State of a bucket -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub enum BucketState { - /// The bucket is deleted - Deleted, - /// The bucket exists - Present(BucketParams), -} - -impl Crdt for BucketState { - fn merge(&mut self, o: &Self) { - match o { - BucketState::Deleted => *self = BucketState::Deleted, - BucketState::Present(other_params) => { - if let BucketState::Present(params) = self { - params.merge(other_params); - } - } - } - } -} - -/// Configuration for a bucket -#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub struct BucketParams { - /// Map of key with access to the bucket, and what kind of access they give - pub authorized_keys: crdt::LwwMap, - /// Is the bucket served as http - pub website: crdt::Lww, -} - -impl Crdt for BucketParams { - fn merge(&mut self, o: &Self) { - self.authorized_keys.merge(&o.authorized_keys); - self.website.merge(&o.website); - } -} - -impl Crdt for Bucket { - fn merge(&mut self, other: &Self) { - self.state.merge(&other.state); - } -} diff --git a/src/model/prev/v051/mod.rs b/src/model/prev/v051/mod.rs deleted file mode 100644 index 8c1335a5..00000000 --- a/src/model/prev/v051/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod bucket_table; diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs index 7b023d87..57eb7b16 100644 --- a/src/model/s3/block_ref_table.rs +++ b/src/model/s3/block_ref_table.rs @@ -3,8 +3,12 @@ use std::sync::Arc; use garage_db as db; use garage_util::data::*; +use garage_util::error::*; +use garage_util::migrate::Migrate; +use garage_block::CalculateRefcount; use garage_table::crdt::Crdt; +use garage_table::replication::TableShardedReplication; use garage_table::*; use garage_block::manager::*; @@ -84,3 +88,38 @@ impl TableSchema for BlockRefTable { filter.apply(entry.deleted.get()) } } + +pub fn block_ref_recount_fn( + block_ref_table: &Arc>, +) -> CalculateRefcount { + let table = Arc::downgrade(block_ref_table); + Box::new(move |tx: &db::Transaction, block: &Hash| { + let table = table + .upgrade() + .ok_or_message("cannot upgrade weak ptr to block_ref_table") + .map_err(db::TxError::Abort)?; + Ok(calculate_refcount(&table, tx, block)?) + }) +} + +fn calculate_refcount( + block_ref_table: &Table, + tx: &db::Transaction, + block: &Hash, +) -> db::TxResult { + let mut result = 0; + for entry in tx.range(&block_ref_table.data.store, block.as_slice()..)? { + let (key, value) = entry?; + if &key[..32] != block.as_slice() { + break; + } + let value = BlockRef::decode(&value) + .ok_or_message("could not decode block_ref") + .map_err(db::TxError::Abort)?; + assert_eq!(value.block, *block); + if !value.deleted.get() { + result += 1; + } + } + Ok(result) +} diff --git a/src/model/s3/lifecycle_worker.rs b/src/model/s3/lifecycle_worker.rs new file mode 100644 index 00000000..af00437e --- /dev/null +++ b/src/model/s3/lifecycle_worker.rs @@ -0,0 +1,414 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::prelude::*; +use std::time::{Duration, Instant}; +use tokio::sync::watch; + +use garage_util::background::*; +use garage_util::data::*; +use garage_util::error::Error; +use garage_util::persister::PersisterShared; +use garage_util::time::*; + +use garage_table::EmptyKey; + +use crate::bucket_table::*; +use crate::s3::object_table::*; + +use crate::garage::Garage; + +mod v090 { + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Default, Clone)] + pub struct LifecycleWorkerPersisted { + pub last_completed: Option, + } + + impl garage_util::migrate::InitialFormat for LifecycleWorkerPersisted { + const VERSION_MARKER: &'static [u8] = b"G09lwp"; + } +} + +pub use v090::*; + +pub struct LifecycleWorker { + garage: Arc, + + state: State, + + persister: PersisterShared, +} + +enum State { + Completed(NaiveDate), + Running { + date: NaiveDate, + pos: Vec, + counter: usize, + objects_expired: usize, + mpu_aborted: usize, + last_bucket: Option, + }, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum Skip { + SkipBucket, + NextObject, +} + +pub fn register_bg_vars( + persister: &PersisterShared, + vars: &mut vars::BgVars, +) { + vars.register_ro(persister, "lifecycle-last-completed", |p| { + p.get_with(|x| x.last_completed.clone().unwrap_or("never".to_string())) + }); +} + +impl LifecycleWorker { + pub fn new(garage: Arc, persister: PersisterShared) -> Self { + let today = today(garage.config.use_local_tz); + let last_completed = persister.get_with(|x| { + x.last_completed + .as_deref() + .and_then(|x| x.parse::().ok()) + }); + let state = match last_completed { + Some(d) if d >= today => State::Completed(d), + _ => State::start(today), + }; + Self { + garage, + state, + persister, + } + } +} + +impl State { + fn start(date: NaiveDate) -> Self { + info!("Starting lifecycle worker for {}", date); + State::Running { + date, + pos: vec![], + counter: 0, + objects_expired: 0, + mpu_aborted: 0, + last_bucket: None, + } + } +} + +#[async_trait] +impl Worker for LifecycleWorker { + fn name(&self) -> String { + "object lifecycle worker".to_string() + } + + fn status(&self) -> WorkerStatus { + match &self.state { + State::Completed(d) => WorkerStatus { + freeform: vec![format!("Last completed: {}", d)], + ..Default::default() + }, + State::Running { + date, + counter, + objects_expired, + mpu_aborted, + .. + } => { + let n_objects = self.garage.object_table.data.store.approximate_len().ok(); + let progress = match n_objects { + Some(total) if total > 0 => format!( + "~{:.2}%", + 100. * std::cmp::min(*counter, total) as f32 / total as f32 + ), + _ => "...".to_string(), + }; + WorkerStatus { + progress: Some(progress), + freeform: vec![ + format!("Started: {}", date), + format!("Objects expired: {}", objects_expired), + format!("Multipart uploads aborted: { }", mpu_aborted), + ], + ..Default::default() + } + } + } + } + + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + match &mut self.state { + State::Completed(_) => Ok(WorkerState::Idle), + State::Running { + date, + counter, + objects_expired, + mpu_aborted, + pos, + last_bucket, + } => { + // Process a batch of 100 items before yielding to bg task scheduler + for _ in 0..100 { + let (object_bytes, next_pos) = match self + .garage + .object_table + .data + .store + .get_gt(&pos)? + { + None => { + info!("Lifecycle worker finished for {}, objects expired: {}, mpu aborted: {}", date, *objects_expired, *mpu_aborted); + self.persister + .set_with(|x| x.last_completed = Some(date.to_string()))?; + self.state = State::Completed(*date); + return Ok(WorkerState::Idle); + } + Some((k, v)) => (v, k), + }; + + let object = self.garage.object_table.data.decode_entry(&object_bytes)?; + let skip = process_object( + &self.garage, + *date, + &object, + objects_expired, + mpu_aborted, + last_bucket, + ) + .await?; + + *counter += 1; + if skip == Skip::SkipBucket { + let bucket_id_len = object.bucket_id.as_slice().len(); + assert_eq!( + next_pos.get(..bucket_id_len), + Some(object.bucket_id.as_slice()) + ); + let last_bucket_pos = [&next_pos[..bucket_id_len], &[0xFFu8][..]].concat(); + *pos = std::cmp::max(next_pos, last_bucket_pos); + } else { + *pos = next_pos; + } + } + + Ok(WorkerState::Busy) + } + } + } + + async fn wait_for_work(&mut self) -> WorkerState { + match &self.state { + State::Completed(d) => { + let use_local_tz = self.garage.config.use_local_tz; + let next_day = d.succ_opt().expect("no next day"); + let next_start = midnight_ts(next_day, use_local_tz); + loop { + let now = now_msec(); + if now < next_start { + tokio::time::sleep_until( + (Instant::now() + Duration::from_millis(next_start - now)).into(), + ) + .await; + } else { + break; + } + } + self.state = State::start(std::cmp::max(next_day, today(use_local_tz))); + } + State::Running { .. } => (), + } + WorkerState::Busy + } +} + +async fn process_object( + garage: &Arc, + now_date: NaiveDate, + object: &Object, + objects_expired: &mut usize, + mpu_aborted: &mut usize, + last_bucket: &mut Option, +) -> Result { + if !object + .versions() + .iter() + .any(|x| x.is_data() || x.is_uploading(None)) + { + return Ok(Skip::NextObject); + } + + let bucket = match last_bucket.take() { + Some(b) if b.id == object.bucket_id => b, + _ => { + match garage + .bucket_table + .get(&EmptyKey, &object.bucket_id) + .await? + { + Some(b) => b, + None => { + warn!( + "Lifecycle worker: object in non-existent bucket {:?}", + object.bucket_id + ); + return Ok(Skip::SkipBucket); + } + } + } + }; + + let lifecycle_policy: &[LifecycleRule] = bucket + .state + .as_option() + .and_then(|s| s.lifecycle_config.get().as_deref()) + .unwrap_or_default(); + + if lifecycle_policy.iter().all(|x| !x.enabled) { + return Ok(Skip::SkipBucket); + } + + let db = garage.object_table.data.store.db(); + + for rule in lifecycle_policy.iter() { + if !rule.enabled { + continue; + } + + if let Some(pfx) = &rule.filter.prefix { + if !object.key.starts_with(pfx) { + continue; + } + } + + if let Some(expire) = &rule.expiration { + if let Some(current_version) = object.versions().iter().rev().find(|v| v.is_data()) { + let version_date = next_date(current_version.timestamp); + + let current_version_data = match ¤t_version.state { + ObjectVersionState::Complete(c) => c, + _ => unreachable!(), + }; + + let size_match = check_size_filter(current_version_data, &rule.filter); + let date_match = match expire { + LifecycleExpiration::AfterDays(n_days) => { + (now_date - version_date) >= chrono::Duration::days(*n_days as i64) + } + LifecycleExpiration::AtDate(exp_date) => { + if let Ok(exp_date) = parse_lifecycle_date(exp_date) { + now_date >= exp_date + } else { + warn!("Invalid expiration date stored in bucket {:?} lifecycle config: {}", bucket.id, exp_date); + false + } + } + }; + + if size_match && date_match { + // Delete expired version + let deleted_object = Object::new( + object.bucket_id, + object.key.clone(), + vec![ObjectVersion { + uuid: gen_uuid(), + timestamp: std::cmp::max(now_msec(), current_version.timestamp + 1), + state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker), + }], + ); + info!( + "Lifecycle: expiring 1 object in bucket {:?}", + object.bucket_id + ); + db.transaction(|tx| garage.object_table.queue_insert(tx, &deleted_object))?; + *objects_expired += 1; + } + } + } + + if let Some(abort_mpu_days) = &rule.abort_incomplete_mpu_days { + let aborted_versions = object + .versions() + .iter() + .filter_map(|v| { + let version_date = next_date(v.timestamp); + if (now_date - version_date) >= chrono::Duration::days(*abort_mpu_days as i64) + && matches!(&v.state, ObjectVersionState::Uploading { .. }) + { + Some(ObjectVersion { + state: ObjectVersionState::Aborted, + ..*v + }) + } else { + None + } + }) + .collect::>(); + if !aborted_versions.is_empty() { + // Insert aborted mpu info + let n_aborted = aborted_versions.len(); + info!( + "Lifecycle: aborting {} incomplete upload(s) in bucket {:?}", + n_aborted, object.bucket_id + ); + let aborted_object = + Object::new(object.bucket_id, object.key.clone(), aborted_versions); + db.transaction(|tx| garage.object_table.queue_insert(tx, &aborted_object))?; + *mpu_aborted += n_aborted; + } + } + } + + *last_bucket = Some(bucket); + Ok(Skip::NextObject) +} + +fn check_size_filter(version_data: &ObjectVersionData, filter: &LifecycleFilter) -> bool { + let size = match version_data { + ObjectVersionData::Inline(meta, _) | ObjectVersionData::FirstBlock(meta, _) => meta.size, + _ => unreachable!(), + }; + if let Some(size_gt) = filter.size_gt { + if !(size > size_gt) { + return false; + } + } + if let Some(size_lt) = filter.size_lt { + if !(size < size_lt) { + return false; + } + } + true +} + +fn midnight_ts(date: NaiveDate, use_local_tz: bool) -> u64 { + let midnight = date.and_hms_opt(0, 0, 0).expect("midnight does not exist"); + if use_local_tz { + return midnight + .and_local_timezone(Local) + .single() + .expect("bad local midnight") + .timestamp_millis() as u64; + } + midnight.and_utc().timestamp_millis() as u64 +} + +fn next_date(ts: u64) -> NaiveDate { + DateTime::::from_timestamp_millis(ts as i64) + .expect("bad timestamp") + .date_naive() + .succ_opt() + .expect("no next day") +} + +fn today(use_local_tz: bool) -> NaiveDate { + if use_local_tz { + return Local::now().naive_local().date(); + } + Utc::now().naive_utc().date() +} diff --git a/src/model/s3/mod.rs b/src/model/s3/mod.rs index 4e94337d..5c776fb0 100644 --- a/src/model/s3/mod.rs +++ b/src/model/s3/mod.rs @@ -1,3 +1,6 @@ pub mod block_ref_table; +pub mod mpu_table; pub mod object_table; pub mod version_table; + +pub mod lifecycle_worker; diff --git a/src/model/s3/mpu_table.rs b/src/model/s3/mpu_table.rs new file mode 100644 index 00000000..c9f79caf --- /dev/null +++ b/src/model/s3/mpu_table.rs @@ -0,0 +1,263 @@ +use std::sync::Arc; + +use garage_db as db; + +use garage_util::crdt::Crdt; +use garage_util::data::*; +use garage_util::time::*; + +use garage_table::replication::TableShardedReplication; +use garage_table::*; + +use crate::index_counter::*; +use crate::s3::version_table::*; + +pub const UPLOADS: &str = "uploads"; +pub const PARTS: &str = "parts"; +pub const BYTES: &str = "bytes"; + +mod v09 { + use crate::s3::object_table::ChecksumValue; + use garage_util::crdt; + use garage_util::data::Uuid; + use serde::{Deserialize, Serialize}; + + /// A part of a multipart upload + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct MultipartUpload { + /// Partition key = Upload id = UUID of the object version + pub upload_id: Uuid, + + /// The timestamp at which the multipart upload was created + pub timestamp: u64, + /// Is this multipart upload deleted + /// The MultipartUpload is marked as deleted as soon as the + /// multipart upload is either completed or aborted + pub deleted: crdt::Bool, + /// List of uploaded parts, key = (part number, timestamp) + /// In case of retries, all versions for each part are kept + /// Everything is cleaned up only once the MultipartUpload is marked deleted + pub parts: crdt::Map, + + // Back link to bucket+key so that we can find the object this mpu + // belongs to and check whether it is still valid + /// Bucket in which the related object is stored + pub bucket_id: Uuid, + /// Key in which the related object is stored + pub key: String, + } + + #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct MpuPartKey { + /// Number of the part + pub part_number: u64, + /// Timestamp of part upload + pub timestamp: u64, + } + + /// The version of an uploaded part + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct MpuPart { + /// Links to a Version in VersionTable + pub version: Uuid, + /// ETag of the content of this part (known only once done uploading) + pub etag: Option, + /// Checksum requested by x-amz-checksum-algorithm + #[serde(default)] + pub checksum: Option, + /// Size of this part (known only once done uploading) + pub size: Option, + } + + impl garage_util::migrate::InitialFormat for MultipartUpload { + const VERSION_MARKER: &'static [u8] = b"G09s3mpu"; + } +} + +pub use v09::*; + +impl Ord for MpuPartKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.part_number + .cmp(&other.part_number) + .then(self.timestamp.cmp(&other.timestamp)) + } +} + +impl PartialOrd for MpuPartKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl MultipartUpload { + pub fn new( + upload_id: Uuid, + timestamp: u64, + bucket_id: Uuid, + key: String, + deleted: bool, + ) -> Self { + Self { + upload_id, + timestamp, + deleted: crdt::Bool::new(deleted), + parts: crdt::Map::new(), + bucket_id, + key, + } + } + + pub fn next_timestamp(&self, part_number: u64) -> u64 { + std::cmp::max( + now_msec(), + 1 + self + .parts + .items() + .iter() + .filter(|(x, _)| x.part_number == part_number) + .map(|(x, _)| x.timestamp) + .max() + .unwrap_or(0), + ) + } +} + +impl Entry for MultipartUpload { + fn partition_key(&self) -> &Uuid { + &self.upload_id + } + fn sort_key(&self) -> &EmptyKey { + &EmptyKey + } + fn is_tombstone(&self) -> bool { + self.deleted.get() + } +} + +impl Crdt for MultipartUpload { + fn merge(&mut self, other: &Self) { + self.deleted.merge(&other.deleted); + + if self.deleted.get() { + self.parts.clear(); + } else { + self.parts.merge(&other.parts); + } + } +} + +impl Crdt for MpuPart { + fn merge(&mut self, other: &Self) { + self.etag = match (self.etag.take(), &other.etag) { + (None, Some(_)) => other.etag.clone(), + (Some(x), Some(y)) if x < *y => other.etag.clone(), + (x, _) => x, + }; + self.size = match (self.size, other.size) { + (None, Some(_)) => other.size, + (Some(x), Some(y)) if x < y => other.size, + (x, _) => x, + }; + self.checksum = match (self.checksum.take(), &other.checksum) { + (None, Some(_)) => other.checksum.clone(), + (Some(x), Some(y)) if x < *y => other.checksum.clone(), + (x, _) => x, + }; + } +} + +pub struct MultipartUploadTable { + pub version_table: Arc>, + pub mpu_counter_table: Arc>, +} + +impl TableSchema for MultipartUploadTable { + const TABLE_NAME: &'static str = "multipart_upload"; + + type P = Uuid; + type S = EmptyKey; + type E = MultipartUpload; + type Filter = DeletedFilter; + + fn updated( + &self, + tx: &mut db::Transaction, + old: Option<&Self::E>, + new: Option<&Self::E>, + ) -> db::TxOpResult<()> { + // 1. Count + let counter_res = self.mpu_counter_table.count(tx, old, new); + if let Err(e) = db::unabort(counter_res)? { + error!( + "Unable to update multipart object part counter: {}. Index values will be wrong!", + e + ); + } + + // 2. Propagate deletions to version table + if let (Some(old_mpu), Some(new_mpu)) = (old, new) { + if new_mpu.deleted.get() && !old_mpu.deleted.get() { + let deleted_versions = old_mpu.parts.items().iter().map(|(_k, p)| { + Version::new( + p.version, + VersionBacklink::MultipartUpload { + upload_id: old_mpu.upload_id, + }, + true, + ) + }); + for version in deleted_versions { + let res = self.version_table.queue_insert(tx, &version); + if let Err(e) = db::unabort(res)? { + error!("Unable to enqueue version deletion propagation: {}. A repair will be needed.", e); + } + } + } + } + + Ok(()) + } + + fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { + filter.apply(entry.is_tombstone()) + } +} + +impl CountedItem for MultipartUpload { + const COUNTER_TABLE_NAME: &'static str = "bucket_mpu_counter"; + + // Partition key = bucket id + type CP = Uuid; + // Sort key = nothing + type CS = EmptyKey; + + fn counter_partition_key(&self) -> &Uuid { + &self.bucket_id + } + fn counter_sort_key(&self) -> &EmptyKey { + &EmptyKey + } + + fn counts(&self) -> Vec<(&'static str, i64)> { + let uploads = if self.deleted.get() { 0 } else { 1 }; + let mut parts = self + .parts + .items() + .iter() + .map(|(k, _)| k.part_number) + .collect::>(); + parts.dedup(); + let bytes = self + .parts + .items() + .iter() + .map(|(_, p)| p.size.unwrap_or(0)) + .sum::(); + vec![ + (UPLOADS, uploads), + (PARTS, parts.len() as i64), + (BYTES, bytes as i64), + ] + } +} diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index 518acc95..6c33b79b 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -10,13 +10,14 @@ use garage_table::replication::TableShardedReplication; use garage_table::*; use crate::index_counter::*; +use crate::s3::mpu_table::*; use crate::s3::version_table::*; pub const OBJECTS: &str = "objects"; pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads"; pub const BYTES: &str = "bytes"; -mod v05 { +mod v08 { use garage_util::data::{Hash, Uuid}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -25,16 +26,16 @@ mod v05 { #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Object { /// The bucket in which the object is stored, used as partition key - pub bucket: String, + pub bucket_id: Uuid, /// The key at which the object is stored in its bucket, used as sorting key pub key: String, - /// The list of currenty stored versions of the object + /// The list of currently stored versions of the object pub(super) versions: Vec, } - /// Informations about a version of an object + /// Information about a version of an object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct ObjectVersion { /// Id of the version @@ -91,16 +92,13 @@ mod v05 { impl garage_util::migrate::InitialFormat for Object {} } -mod v08 { +mod v09 { use garage_util::data::Uuid; use serde::{Deserialize, Serialize}; - use super::v05; + use super::v08; - pub use v05::{ - ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta, - ObjectVersionState, - }; + pub use v08::{ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta}; /// An object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] @@ -111,26 +109,269 @@ mod v08 { /// The key at which the object is stored in its bucket, used as sorting key pub key: String, - /// The list of currenty stored versions of the object + /// The list of currently stored versions of the object pub(super) versions: Vec, } + /// Information about a version of an object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersion { + /// Id of the version + pub uuid: Uuid, + /// Timestamp of when the object was created + pub timestamp: u64, + /// State of the version + pub state: ObjectVersionState, + } + + /// State of an object version + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionState { + /// The version is being received + Uploading { + /// Indicates whether this is a multipart upload + multipart: bool, + /// Headers to be included in the final object + headers: ObjectVersionHeaders, + }, + /// The version is fully received + Complete(ObjectVersionData), + /// The version uploaded containded errors or the upload was explicitly aborted + Aborted, + } + impl garage_util::migrate::Migrate for Object { - type Previous = v05::Object; + const VERSION_MARKER: &'static [u8] = b"G09s3o"; - fn migrate(old: v05::Object) -> Object { - use garage_util::data::blake2sum; + type Previous = v08::Object; + fn migrate(old: v08::Object) -> Object { + let versions = old + .versions + .into_iter() + .map(|x| ObjectVersion { + uuid: x.uuid, + timestamp: x.timestamp, + state: match x.state { + v08::ObjectVersionState::Uploading(h) => ObjectVersionState::Uploading { + multipart: false, + headers: h, + }, + v08::ObjectVersionState::Complete(d) => ObjectVersionState::Complete(d), + v08::ObjectVersionState::Aborted => ObjectVersionState::Aborted, + }, + }) + .collect(); Object { - bucket_id: blake2sum(old.bucket.as_bytes()), + bucket_id: old.bucket_id, key: old.key, - versions: old.versions, + versions, } } } } -pub use v08::*; +mod v010 { + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + + use super::v09; + + /// An object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct Object { + /// The bucket in which the object is stored, used as partition key + pub bucket_id: Uuid, + + /// The key at which the object is stored in its bucket, used as sorting key + pub key: String, + + /// The list of currently stored versions of the object + pub(super) versions: Vec, + } + + /// Information about a version of an object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersion { + /// Id of the version + pub uuid: Uuid, + /// Timestamp of when the object was created + pub timestamp: u64, + /// State of the version + pub state: ObjectVersionState, + } + + /// State of an object version + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionState { + /// The version is being received + Uploading { + /// Indicates whether this is a multipart upload + multipart: bool, + /// Checksum algorithm to use + checksum_algorithm: Option, + /// Encryption params + headers to be included in the final object + encryption: ObjectVersionEncryption, + }, + /// The version is fully received + Complete(ObjectVersionData), + /// The version uploaded containded errors or the upload was explicitly aborted + Aborted, + } + + /// Data stored in object version + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionData { + /// The object was deleted, this Version is a tombstone to mark it as such + DeleteMarker, + /// The object is short, it's stored inlined. + /// It is never compressed. For encrypted objects, it is encrypted using + /// AES256-GCM, like the encrypted headers. + Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec), + /// The object is not short, Hash of first block is stored here, next segments hashes are + /// stored in the version table + FirstBlock(ObjectVersionMeta, Hash), + } + + /// Metadata about the object version + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersionMeta { + /// Size of the object. If object is encrypted/compressed, + /// this is always the size of the unencrypted/uncompressed data + pub size: u64, + /// etag of the object + pub etag: String, + /// Encryption params + headers (encrypted or plaintext) + pub encryption: ObjectVersionEncryption, + } + + /// Encryption information + metadata + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub enum ObjectVersionEncryption { + SseC { + /// Encrypted serialized ObjectVersionInner struct. + /// This is never compressed, just encrypted using AES256-GCM. + #[serde(with = "serde_bytes")] + inner: Vec, + /// Whether data blocks are compressed in addition to being encrypted + /// (compression happens before encryption, whereas for non-encrypted + /// objects, compression is handled at the level of the block manager) + compressed: bool, + }, + Plaintext { + /// Plain-text headers + inner: ObjectVersionMetaInner, + }, + } + + /// Vector of headers, as tuples of the format (header name, header value) + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct ObjectVersionMetaInner { + pub headers: HeaderList, + pub checksum: Option, + } + + pub type HeaderList = Vec<(String, String)>; + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ChecksumAlgorithm { + Crc32, + Crc32c, + Sha1, + Sha256, + } + + /// Checksum value for x-amz-checksum-algorithm + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ChecksumValue { + Crc32(#[serde(with = "serde_bytes")] [u8; 4]), + Crc32c(#[serde(with = "serde_bytes")] [u8; 4]), + Sha1(#[serde(with = "serde_bytes")] [u8; 20]), + Sha256(#[serde(with = "serde_bytes")] [u8; 32]), + } + + impl garage_util::migrate::Migrate for Object { + const VERSION_MARKER: &'static [u8] = b"G010s3ob"; + + type Previous = v09::Object; + + fn migrate(old: v09::Object) -> Object { + Object { + bucket_id: old.bucket_id, + key: old.key, + versions: old.versions.into_iter().map(migrate_version).collect(), + } + } + } + + fn migrate_version(old: v09::ObjectVersion) -> ObjectVersion { + ObjectVersion { + uuid: old.uuid, + timestamp: old.timestamp, + state: match old.state { + v09::ObjectVersionState::Uploading { multipart, headers } => { + ObjectVersionState::Uploading { + multipart, + checksum_algorithm: None, + encryption: migrate_headers(headers), + } + } + v09::ObjectVersionState::Complete(d) => { + ObjectVersionState::Complete(migrate_data(d)) + } + v09::ObjectVersionState::Aborted => ObjectVersionState::Aborted, + }, + } + } + + fn migrate_data(old: v09::ObjectVersionData) -> ObjectVersionData { + match old { + v09::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker, + v09::ObjectVersionData::Inline(meta, data) => { + ObjectVersionData::Inline(migrate_meta(meta), data) + } + v09::ObjectVersionData::FirstBlock(meta, fb) => { + ObjectVersionData::FirstBlock(migrate_meta(meta), fb) + } + } + } + + fn migrate_meta(old: v09::ObjectVersionMeta) -> ObjectVersionMeta { + ObjectVersionMeta { + size: old.size, + etag: old.etag, + encryption: migrate_headers(old.headers), + } + } + + fn migrate_headers(old: v09::ObjectVersionHeaders) -> ObjectVersionEncryption { + use http::header::CONTENT_TYPE; + + let mut new_headers = Vec::with_capacity(old.other.len() + 1); + if old.content_type != "blob" { + new_headers.push((CONTENT_TYPE.as_str().to_string(), old.content_type)); + } + for (name, value) in old.other.into_iter() { + new_headers.push((name, value)); + } + + ObjectVersionEncryption::Plaintext { + inner: ObjectVersionMetaInner { + headers: new_headers, + checksum: None, + }, + } + } + + // Since ObjectVersionMetaInner can now be serialized independently, for the + // purpose of being encrypted, we need it to support migrations on its own + // as well. + impl garage_util::migrate::InitialFormat for ObjectVersionMetaInner { + const VERSION_MARKER: &'static [u8] = b"G010s3om"; + } +} + +pub use v010::*; impl Object { /// Initialize an Object struct from parts @@ -180,11 +421,11 @@ impl Crdt for ObjectVersionState { Complete(a) => { a.merge(b); } - Uploading(_) => { + Uploading { .. } => { *self = Complete(b.clone()); } }, - Uploading(_) => {} + Uploading { .. } => {} } } } @@ -199,8 +440,17 @@ impl ObjectVersion { } /// Is the object version currently being uploaded - pub fn is_uploading(&self) -> bool { - matches!(self.state, ObjectVersionState::Uploading(_)) + /// + /// matches only multipart uploads if check_multipart is Some(true) + /// matches only non-multipart uploads if check_multipart is Some(false) + /// matches both if check_multipart is None + pub fn is_uploading(&self, check_multipart: Option) -> bool { + match &self.state { + ObjectVersionState::Uploading { multipart, .. } => { + check_multipart.map(|x| x == *multipart).unwrap_or(true) + } + _ => false, + } } /// Is the object version completely received @@ -232,6 +482,17 @@ impl Entry for Object { } } +impl ChecksumValue { + pub fn algorithm(&self) -> ChecksumAlgorithm { + match self { + ChecksumValue::Crc32(_) => ChecksumAlgorithm::Crc32, + ChecksumValue::Crc32c(_) => ChecksumAlgorithm::Crc32c, + ChecksumValue::Sha1(_) => ChecksumAlgorithm::Sha1, + ChecksumValue::Sha256(_) => ChecksumAlgorithm::Sha256, + } + } +} + impl Crdt for Object { fn merge(&mut self, other: &Self) { // Merge versions from other into here @@ -267,13 +528,20 @@ impl Crdt for Object { pub struct ObjectTable { pub version_table: Arc>, + pub mpu_table: Arc>, pub object_counter_table: Arc>, } #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub enum ObjectFilter { + /// Is the object version available (received and not a tombstone) IsData, - IsUploading, + /// Is the object version currently being uploaded + /// + /// matches only multipart uploads if check_multipart is Some(true) + /// matches only non-multipart uploads if check_multipart is Some(false) + /// matches both if check_multipart is None + IsUploading { check_multipart: Option }, } impl TableSchema for ObjectTable { @@ -301,21 +569,28 @@ impl TableSchema for ObjectTable { // 2. Enqueue propagation deletions to version table if let (Some(old_v), Some(new_v)) = (old, new) { - // Propagate deletion of old versions for v in old_v.versions.iter() { - let newly_deleted = match new_v + let new_v_id = new_v .versions - .binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key())) - { + .binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key())); + + // Propagate deletion of old versions to the Version table + let delete_version = match new_v_id { Err(_) => true, Ok(i) => { new_v.versions[i].state == ObjectVersionState::Aborted && v.state != ObjectVersionState::Aborted } }; - if newly_deleted { - let deleted_version = - Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true); + if delete_version { + let deleted_version = Version::new( + v.uuid, + VersionBacklink::Object { + bucket_id: old_v.bucket_id, + key: old_v.key.clone(), + }, + true, + ); let res = self.version_table.queue_insert(tx, &deleted_version); if let Err(e) = db::unabort(res)? { error!( @@ -324,6 +599,39 @@ impl TableSchema for ObjectTable { ); } } + + // After abortion or completion of multipart uploads, delete MPU table entry + if matches!( + v.state, + ObjectVersionState::Uploading { + multipart: true, + .. + } + ) { + let delete_mpu = match new_v_id { + Err(_) => true, + Ok(i) => !matches!( + new_v.versions[i].state, + ObjectVersionState::Uploading { .. } + ), + }; + if delete_mpu { + let deleted_mpu = MultipartUpload::new( + v.uuid, + v.timestamp, + old_v.bucket_id, + old_v.key.clone(), + true, + ); + let res = self.mpu_table.queue_insert(tx, &deleted_mpu); + if let Err(e) = db::unabort(res)? { + error!( + "Unable to enqueue multipart upload deletion propagation: {}. A repair will be needed.", + e + ); + } + } + } } } @@ -333,7 +641,10 @@ impl TableSchema for ObjectTable { fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { match filter { ObjectFilter::IsData => entry.versions.iter().any(|v| v.is_data()), - ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()), + ObjectFilter::IsUploading { check_multipart } => entry + .versions + .iter() + .any(|v| v.is_uploading(*check_multipart)), } } } @@ -360,10 +671,7 @@ impl CountedItem for Object { } else { 0 }; - let n_unfinished_uploads = versions - .iter() - .filter(|v| matches!(v.state, ObjectVersionState::Uploading(_))) - .count(); + let n_unfinished_uploads = versions.iter().filter(|v| v.is_uploading(None)).count(); let n_bytes = versions .iter() .map(|v| match &v.state { diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index 6edc83f4..45be5af8 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use garage_db as db; use garage_util::data::*; +use garage_util::error::*; use garage_table::crdt::*; use garage_table::replication::TableShardedReplication; @@ -10,62 +11,11 @@ use garage_table::*; use crate::s3::block_ref_table::*; -mod v05 { +mod v08 { use garage_util::crdt; use garage_util::data::{Hash, Uuid}; use serde::{Deserialize, Serialize}; - /// A version of an object - #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] - pub struct Version { - /// UUID of the version, used as partition key - pub uuid: Uuid, - - // Actual data: the blocks for this version - // In the case of a multipart upload, also store the etags - // of individual parts and check them when doing CompleteMultipartUpload - /// Is this version deleted - pub deleted: crdt::Bool, - /// list of blocks of data composing the version - pub blocks: crdt::Map, - /// Etag of each part in case of a multipart upload, empty otherwise - pub parts_etags: crdt::Map, - - // Back link to bucket+key so that we can figure if - // this was deleted later on - /// Bucket in which the related object is stored - pub bucket: String, - /// Key in which the related object is stored - pub key: String, - } - - #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] - pub struct VersionBlockKey { - /// Number of the part - pub part_number: u64, - /// Offset of this sub-segment in its part - pub offset: u64, - } - - /// Informations about a single block - #[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)] - pub struct VersionBlock { - /// Blake2 sum of the block - pub hash: Hash, - /// Size of the block - pub size: u64, - } - - impl garage_util::migrate::InitialFormat for Version {} -} - -mod v08 { - use garage_util::crdt; - use garage_util::data::Uuid; - use serde::{Deserialize, Serialize}; - - use super::v05; - /// A version of an object #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct Version { @@ -90,52 +40,115 @@ mod v08 { pub key: String, } - pub use v05::{VersionBlock, VersionBlockKey}; + #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct VersionBlockKey { + /// Number of the part + pub part_number: u64, + /// Offset of this sub-segment in its part as sent by the client + /// (before any kind of compression or encryption) + pub offset: u64, + } + + /// Information about a single block + #[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct VersionBlock { + /// Blake2 sum of the block + pub hash: Hash, + /// Size of the block, before any kind of compression or encryption + pub size: u64, + } + + impl garage_util::migrate::InitialFormat for Version {} +} + +pub(crate) mod v09 { + use garage_util::crdt; + use garage_util::data::Uuid; + use serde::{Deserialize, Serialize}; + + use super::v08; + + pub use v08::{VersionBlock, VersionBlockKey}; + + /// A version of an object + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub struct Version { + /// UUID of the version, used as partition key + pub uuid: Uuid, + + // Actual data: the blocks for this version + // In the case of a multipart upload, also store the etags + // of individual parts and check them when doing CompleteMultipartUpload + /// Is this version deleted + pub deleted: crdt::Bool, + /// list of blocks of data composing the version + pub blocks: crdt::Map, + + // Back link to owner of this version (either an object or a multipart + // upload), used to find whether it has been deleted and this version + // should in turn be deleted (see versions repair procedure) + pub backlink: VersionBacklink, + } + + #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] + pub enum VersionBacklink { + Object { + /// Bucket in which the related object is stored + bucket_id: Uuid, + /// Key in which the related object is stored + key: String, + }, + MultipartUpload { + upload_id: Uuid, + }, + } impl garage_util::migrate::Migrate for Version { - type Previous = v05::Version; + const VERSION_MARKER: &'static [u8] = b"G09s3v"; - fn migrate(old: v05::Version) -> Version { - use garage_util::data::blake2sum; + type Previous = v08::Version; + fn migrate(old: v08::Version) -> Version { Version { uuid: old.uuid, deleted: old.deleted, blocks: old.blocks, - parts_etags: old.parts_etags, - bucket_id: blake2sum(old.bucket.as_bytes()), - key: old.key, + backlink: VersionBacklink::Object { + bucket_id: old.bucket_id, + key: old.key, + }, } } } } -pub use v08::*; +pub use v09::*; impl Version { - pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self { + pub fn new(uuid: Uuid, backlink: VersionBacklink, deleted: bool) -> Self { Self { uuid, deleted: deleted.into(), blocks: crdt::Map::new(), - parts_etags: crdt::Map::new(), - bucket_id, - key, + backlink, } } pub fn has_part_number(&self, part_number: u64) -> bool { - let case1 = self - .parts_etags - .items() - .binary_search_by(|(k, _)| k.cmp(&part_number)) - .is_ok(); - let case2 = self - .blocks + self.blocks .items() .binary_search_by(|(k, _)| k.part_number.cmp(&part_number)) - .is_ok(); - case1 || case2 + .is_ok() + } + + pub fn n_parts(&self) -> Result { + Ok(self + .blocks + .items() + .last() + .ok_or_message("version has no parts")? + .0 + .part_number) } } @@ -175,10 +188,8 @@ impl Crdt for Version { if self.deleted.get() { self.blocks.clear(); - self.parts_etags.clear(); } else { self.blocks.merge(&other.blocks); - self.parts_etags.merge(&other.parts_etags); } } } diff --git a/src/model/snapshot.rs b/src/model/snapshot.rs new file mode 100644 index 00000000..8e8995f9 --- /dev/null +++ b/src/model/snapshot.rs @@ -0,0 +1,142 @@ +use std::fs; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use rand::prelude::*; +use tokio::sync::watch; + +use garage_util::background::*; +use garage_util::error::*; + +use crate::garage::Garage; + +// The two most recent snapshots are kept +const KEEP_SNAPSHOTS: usize = 2; + +static SNAPSHOT_MUTEX: Mutex<()> = Mutex::new(()); + +// ================ snapshotting logic ===================== + +/// Run snapshot_metadata in a blocking thread and async await on it +pub async fn async_snapshot_metadata(garage: &Arc) -> Result<(), Error> { + let garage = garage.clone(); + let worker = tokio::task::spawn_blocking(move || snapshot_metadata(&garage)); + worker.await.unwrap()?; + Ok(()) +} + +/// Take a snapshot of the metadata database, and erase older +/// snapshots if necessary. +/// This is not an async function, it should be spawned on a thread pool +pub fn snapshot_metadata(garage: &Garage) -> Result<(), Error> { + let lock = match SNAPSHOT_MUTEX.try_lock() { + Ok(lock) => lock, + Err(_) => { + return Err(Error::Message( + "Cannot acquire lock, another snapshot might be in progress".into(), + )) + } + }; + + let snapshots_dir = match &garage.config.metadata_snapshots_dir { + Some(d) => d.clone(), + None => { + let mut default_snapshots_dir = garage.config.metadata_dir.clone(); + default_snapshots_dir.push("snapshots"); + default_snapshots_dir + } + }; + fs::create_dir_all(&snapshots_dir)?; + + let mut new_path = snapshots_dir.clone(); + new_path.push(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)); + + info!("Snapshotting metadata db to {}", new_path.display()); + garage.db.snapshot(&new_path)?; + info!("Metadata db snapshot finished"); + + if let Err(e) = cleanup_snapshots(&snapshots_dir) { + error!("Failed to do cleanup in snapshots directory: {}", e); + } + + drop(lock); + + Ok(()) +} + +fn cleanup_snapshots(snapshots_dir: &PathBuf) -> Result<(), Error> { + let mut snapshots = + fs::read_dir(&snapshots_dir)?.collect::, std::io::Error>>()?; + + snapshots.retain(|x| x.file_name().len() > 8); + snapshots.sort_by_key(|x| x.file_name()); + + for to_delete in snapshots.iter().rev().skip(KEEP_SNAPSHOTS) { + let path = snapshots_dir.join(to_delete.path()); + if to_delete.metadata()?.file_type().is_dir() { + for file in fs::read_dir(&path)? { + let file = file?; + if file.metadata()?.is_file() { + fs::remove_file(path.join(file.path()))?; + } + } + std::fs::remove_dir(&path)?; + } else { + std::fs::remove_file(&path)?; + } + } + Ok(()) +} + +// ================ auto snapshot worker ===================== + +pub struct AutoSnapshotWorker { + garage: Arc, + next_snapshot: Instant, + snapshot_interval: Duration, +} + +impl AutoSnapshotWorker { + pub(crate) fn new(garage: Arc, snapshot_interval: Duration) -> Self { + Self { + garage, + snapshot_interval, + next_snapshot: Instant::now() + (snapshot_interval / 2), + } + } +} + +#[async_trait] +impl Worker for AutoSnapshotWorker { + fn name(&self) -> String { + "Metadata snapshot worker".into() + } + fn status(&self) -> WorkerStatus { + WorkerStatus { + freeform: vec![format!( + "Next snapshot: {}", + (chrono::Utc::now() + (self.next_snapshot - Instant::now())).to_rfc3339() + )], + ..Default::default() + } + } + async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { + if Instant::now() < self.next_snapshot { + return Ok(WorkerState::Idle); + } + + async_snapshot_metadata(&self.garage).await?; + + let rand_factor = 1f32 + thread_rng().gen::() / 5f32; + self.next_snapshot = Instant::now() + self.snapshot_interval.mul_f32(rand_factor); + + Ok(WorkerState::Idle) + } + async fn wait_for_work(&mut self) -> WorkerState { + tokio::time::sleep_until(self.next_snapshot.into()).await; + WorkerState::Busy + } +} diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml new file mode 100644 index 00000000..71f42c68 --- /dev/null +++ b/src/net/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "garage_net" +version = "1.3.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Networking library for Garage RPC communication, forked from Netapp" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +[features] +default = [] +telemetry = ["opentelemetry", "opentelemetry-contrib"] + +[dependencies] +futures.workspace = true +pin-project.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tokio-stream.workspace = true + +serde.workspace = true +rmp-serde.workspace = true +hex.workspace = true + +rand.workspace = true + +log.workspace = true +arc-swap.workspace = true +thiserror.workspace = true +bytes.workspace = true +cfg-if.workspace = true + +sodiumoxide.workspace = true +kuska-handshake.workspace = true + +opentelemetry = { workspace = true, optional = true } +opentelemetry-contrib = { workspace = true, optional = true } + +[dev-dependencies] +pretty_env_logger.workspace = true diff --git a/src/net/bytes_buf.rs b/src/net/bytes_buf.rs new file mode 100644 index 00000000..1d928ffb --- /dev/null +++ b/src/net/bytes_buf.rs @@ -0,0 +1,199 @@ +use std::cmp::Ordering; +use std::collections::VecDeque; + +use bytes::BytesMut; + +use crate::stream::ByteStream; + +pub use bytes::Bytes; + +/// A circular buffer of bytes, internally represented as a list of Bytes +/// for optimization, but that for all intent and purposes acts just like +/// a big byte slice which can be extended on the right and from which +/// stuff can be taken on the left. +pub struct BytesBuf { + buf: VecDeque, + buf_len: usize, +} + +impl BytesBuf { + /// Creates a new empty BytesBuf + pub fn new() -> Self { + Self { + buf: VecDeque::new(), + buf_len: 0, + } + } + + /// Returns the number of bytes stored in the BytesBuf + #[inline] + pub fn len(&self) -> usize { + self.buf_len + } + + /// Returns true iff the BytesBuf contains zero bytes + #[inline] + pub fn is_empty(&self) -> bool { + self.buf_len == 0 + } + + /// Adds some bytes to the right of the buffer + pub fn extend(&mut self, b: Bytes) { + if !b.is_empty() { + self.buf_len += b.len(); + self.buf.push_back(b); + } + } + + /// Takes the whole content of the buffer and returns it as a single Bytes unit + pub fn take_all(&mut self) -> Bytes { + if self.buf.is_empty() { + Bytes::new() + } else if self.buf.len() == 1 { + self.buf_len = 0; + self.buf.pop_back().unwrap() + } else { + let mut ret = BytesMut::with_capacity(self.buf_len); + for b in self.buf.iter() { + ret.extend_from_slice(&b[..]); + } + self.buf.clear(); + self.buf_len = 0; + ret.freeze() + } + } + + /// Takes at most max_len bytes from the left of the buffer + pub fn take_max(&mut self, max_len: usize) -> Bytes { + if self.buf_len <= max_len { + self.take_all() + } else { + self.take_exact_ok(max_len) + } + } + + /// Take exactly len bytes from the left of the buffer, returns None if + /// the BytesBuf doesn't contain enough data + pub fn take_exact(&mut self, len: usize) -> Option { + if self.buf_len < len { + None + } else { + Some(self.take_exact_ok(len)) + } + } + + fn take_exact_ok(&mut self, len: usize) -> Bytes { + assert!(len <= self.buf_len); + let front = self.buf.pop_front().unwrap(); + match front.len().cmp(&len) { + Ordering::Greater => { + self.buf.push_front(front.slice(len..)); + self.buf_len -= len; + front.slice(..len) + } + Ordering::Equal => { + self.buf_len -= len; + front + } + Ordering::Less => { + let mut ret = BytesMut::with_capacity(len); + ret.extend_from_slice(&front[..]); + self.buf_len -= front.len(); + while ret.len() < len { + let front = self.buf.pop_front().unwrap(); + if front.len() > len - ret.len() { + let take = len - ret.len(); + ret.extend_from_slice(&front[..take]); + self.buf.push_front(front.slice(take..)); + self.buf_len -= take; + break; + } else { + ret.extend_from_slice(&front[..]); + self.buf_len -= front.len(); + } + } + ret.freeze() + } + } + } + + /// Return the internal sequence of Bytes slices that make up the buffer + pub fn into_slices(self) -> VecDeque { + self.buf + } + + /// Return the entire buffer concatenated into a single big Bytes + pub fn into_bytes(mut self) -> Bytes { + self.take_all() + } + + /// Return the content as a stream of individual chunks + pub fn into_stream(self) -> ByteStream { + use futures::stream::StreamExt; + Box::pin(futures::stream::iter(self.buf).map(|x| Ok(x))) + } +} + +impl Default for BytesBuf { + fn default() -> Self { + Self::new() + } +} + +impl From for BytesBuf { + fn from(b: Bytes) -> BytesBuf { + let mut ret = BytesBuf::new(); + ret.extend(b); + ret + } +} + +impl From for Bytes { + fn from(mut b: BytesBuf) -> Bytes { + b.take_all() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bytes_buf() { + let mut buf = BytesBuf::new(); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + + buf.extend(Bytes::from(b"Hello, world!".to_vec())); + assert!(buf.len() == 13); + assert!(!buf.is_empty()); + + buf.extend(Bytes::from(b"1234567890".to_vec())); + assert!(buf.len() == 23); + assert!(!buf.is_empty()); + + assert_eq!( + buf.take_all(), + Bytes::from(b"Hello, world!1234567890".to_vec()) + ); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + + buf.extend(Bytes::from(b"1234567890".to_vec())); + buf.extend(Bytes::from(b"Hello, world!".to_vec())); + assert!(buf.len() == 23); + assert!(!buf.is_empty()); + + assert_eq!(buf.take_max(12), Bytes::from(b"1234567890He".to_vec())); + assert!(buf.len() == 11); + + assert_eq!(buf.take_exact(12), None); + assert!(buf.len() == 11); + assert_eq!( + buf.take_exact(11), + Some(Bytes::from(b"llo, world!".to_vec())) + ); + assert!(buf.len() == 0); + assert!(buf.is_empty()); + } +} diff --git a/src/net/client.rs b/src/net/client.rs new file mode 100644 index 00000000..20e1dacd --- /dev/null +++ b/src/net/client.rs @@ -0,0 +1,290 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::atomic::{self, AtomicU32}; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + +use arc_swap::ArcSwapOption; +use bytes::Bytes; +use log::{debug, error, trace}; + +use futures::io::AsyncReadExt; +use futures::Stream; +use kuska_handshake::async_std::{handshake_client, BoxStream}; +use tokio::net::TcpStream; +use tokio::select; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio_util::compat::*; + +#[cfg(feature = "telemetry")] +use opentelemetry::{ + trace::{FutureExt, Span, SpanKind, TraceContextExt, Tracer}, + Context, KeyValue, +}; +#[cfg(feature = "telemetry")] +use opentelemetry_contrib::trace::propagator::binary::*; + +use crate::error::*; +use crate::message::*; +use crate::netapp::*; +use crate::recv::*; +use crate::send::*; +use crate::stream::*; +use crate::util::*; + +pub(crate) struct ClientConn { + pub(crate) remote_addr: SocketAddr, + pub(crate) peer_id: NodeID, + + query_send: ArcSwapOption>, + + next_query_number: AtomicU32, + inflight: Mutex>>, +} + +impl ClientConn { + pub(crate) async fn init( + netapp: Arc, + socket: TcpStream, + peer_id: NodeID, + ) -> Result<(), Error> { + let remote_addr = socket.peer_addr()?; + let mut socket = socket.compat(); + + // Do handshake to authenticate and prove our identity to server + let handshake = handshake_client( + &mut socket, + netapp.netid.clone(), + netapp.id, + netapp.privkey.clone(), + peer_id, + ) + .await?; + + debug!( + "Handshake complete (client) with {}@{}", + hex::encode(peer_id), + remote_addr + ); + + // Create BoxStream layer that encodes content + let (read, write) = socket.split(); + let (mut read, write) = + BoxStream::from_handshake(read, write, handshake, 0x8000).split_read_write(); + + // Before doing anything, receive version tag and + // check they are running the same version as us + let mut their_version_tag = VersionTag::default(); + read.read_exact(&mut their_version_tag[..]).await?; + if their_version_tag != netapp.version_tag { + let msg = format!( + "different version tags: {} (theirs) vs. {} (ours)", + hex::encode(their_version_tag), + hex::encode(netapp.version_tag) + ); + error!("Cannot connect to {}: {}", hex::encode(&peer_id[..8]), msg); + return Err(Error::VersionMismatch(msg)); + } + + // Build and launch stuff that manages sending requests client-side + let (query_send, query_recv) = mpsc::unbounded_channel(); + + let (stop_recv_loop, stop_recv_loop_recv) = watch::channel(false); + + let conn = Arc::new(ClientConn { + remote_addr, + peer_id, + next_query_number: AtomicU32::from(RequestID::default()), + query_send: ArcSwapOption::new(Some(Arc::new(query_send))), + inflight: Mutex::new(HashMap::new()), + }); + + netapp.connected_as_client(peer_id, conn.clone()); + + let debug_name = format!("CLI {}", hex::encode(&peer_id[..8])); + + tokio::spawn(async move { + let debug_name_2 = debug_name.clone(); + let send_future = tokio::spawn(conn.clone().send_loop(query_recv, write, debug_name_2)); + + let conn2 = conn.clone(); + let recv_future = tokio::spawn(async move { + select! { + r = conn2.recv_loop(read, debug_name) => r, + _ = await_exit(stop_recv_loop_recv) => Ok(()) + } + }); + + send_future.await.log_err("ClientConn send_loop"); + + // FIXME: should do here: wait for inflight requests to all have their response + stop_recv_loop + .send(true) + .log_err("ClientConn send true to stop_recv_loop"); + + recv_future.await.log_err("ClientConn recv_loop"); + + // Make sure we don't wait on any more requests that won't + // have a response + conn.inflight.lock().unwrap().clear(); + + netapp.disconnected_as_client(&peer_id, conn); + }); + + Ok(()) + } + + pub fn close(&self) { + self.query_send.store(None); + } + + pub(crate) async fn call( + self: Arc, + req: Req, + path: &str, + prio: RequestPriority, + ) -> Result, Error> + where + T: Message, + { + let query_send = self.query_send.load_full().ok_or(Error::ConnectionClosed)?; + + let id = self + .next_query_number + .fetch_add(1, atomic::Ordering::Relaxed); + + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let tracer = opentelemetry::global::tracer("netapp"); + let mut span = tracer.span_builder(format!("RPC >> {}", path)) + .with_kind(SpanKind::Client) + .start(&tracer); + let propagator = BinaryPropagator::new(); + let telemetry_id: Bytes = propagator.to_bytes(span.span_context()).to_vec().into(); + } else { + let telemetry_id: Bytes = Bytes::new(); + } + }; + + // Encode request + let req_enc = req.into_enc(prio, path.as_bytes().to_vec().into(), telemetry_id); + let req_msg_len = req_enc.msg.len(); + let (req_stream, req_order) = req_enc.encode(); + + // Send request through + let (resp_send, resp_recv) = oneshot::channel(); + let old = self.inflight.lock().unwrap().insert(id, resp_send); + if let Some(old_ch) = old { + error!( + "Too many inflight requests! RequestID collision. Interrupting previous request." + ); + let _ = old_ch.send(Box::pin(futures::stream::once(async move { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "RequestID collision, too many inflight requests", + )) + }))); + } + + debug!( + "request: query_send {}, path {}, prio {} (serialized message: {} bytes)", + id, path, prio, req_msg_len + ); + + #[cfg(feature = "telemetry")] + span.set_attribute(KeyValue::new("len_query_msg", req_msg_len as i64)); + + query_send.send(SendItem::Stream(id, prio, req_order, req_stream))?; + + let canceller = CancelOnDrop::new(id, query_send.as_ref().clone()); + + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let stream = resp_recv + .with_context(Context::current_with_span(span)) + .await?; + } else { + let stream = resp_recv.await?; + } + } + + let stream = Box::pin(canceller.for_stream(stream)); + + let resp_enc = RespEnc::decode(stream).await?; + debug!("client: got response to request {} (path {})", id, path); + Resp::from_enc(resp_enc) + } +} + +impl SendLoop for ClientConn {} + +impl RecvLoop for ClientConn { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream) { + trace!("ClientConn recv_handler {}", id); + + let mut inflight = self.inflight.lock().unwrap(); + if let Some(ch) = inflight.remove(&id) { + if ch.send(stream).is_err() { + debug!("Could not send request response, probably because request was interrupted. Dropping response."); + } + } else { + debug!("Got unexpected response to request {}, dropping it", id); + } + } +} + +// ---- + +struct CancelOnDrop { + id: RequestID, + query_send: mpsc::UnboundedSender, +} + +impl CancelOnDrop { + fn new(id: RequestID, query_send: mpsc::UnboundedSender) -> Self { + Self { id, query_send } + } + fn for_stream(self, stream: ByteStream) -> CancelOnDropStream { + CancelOnDropStream { + cancel: Some(self), + stream, + } + } +} + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + trace!("cancelling request {}", self.id); + let _ = self.query_send.send(SendItem::Cancel(self.id)); + } +} + +#[pin_project::pin_project] +struct CancelOnDropStream { + cancel: Option, + #[pin] + stream: ByteStream, +} + +impl Stream for CancelOnDropStream { + type Item = Packet; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + let this = self.project(); + let res = this.stream.poll_next(cx); + if matches!(res, Poll::Ready(None)) { + if let Some(c) = this.cancel.take() { + std::mem::forget(c) + } + } + res + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} diff --git a/src/net/endpoint.rs b/src/net/endpoint.rs new file mode 100644 index 00000000..3ab1048a --- /dev/null +++ b/src/net/endpoint.rs @@ -0,0 +1,199 @@ +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use arc_swap::ArcSwapOption; +use futures::future::{BoxFuture, FutureExt}; + +use crate::error::Error; +use crate::message::*; +use crate::netapp::*; + +/// This trait should be implemented by an object of your application +/// that can handle a message of type `M`, if it wishes to handle +/// streams attached to the request and/or to send back streams +/// attached to the response.. +/// +/// The handler object should be in an Arc, see `Endpoint::set_handler` +pub trait StreamingEndpointHandler: Send + Sync +where + M: Message, +{ + fn handle(self: &Arc, m: Req, from: NodeID) -> impl Future> + Send; +} + +/// If one simply wants to use an endpoint in a client fashion, +/// without locally serving requests to that endpoint, +/// use the unit type `()` as the handler type: +/// it will panic if it is ever made to handle request. +impl EndpointHandler for () { + async fn handle(self: &Arc<()>, _m: &M, _from: NodeID) -> M::Response { + panic!("This endpoint should not have a local handler."); + } +} + +// ---- + +/// This trait should be implemented by an object of your application +/// that can handle a message of type `M`, in the cases where it doesn't +/// care about attached stream in the request nor in the response. +pub trait EndpointHandler: Send + Sync +where + M: Message, +{ + fn handle(self: &Arc, m: &M, from: NodeID) -> impl Future + Send; +} + +impl StreamingEndpointHandler for T +where + T: EndpointHandler, + M: Message, +{ + async fn handle(self: &Arc, mut m: Req, from: NodeID) -> Resp { + // Immediately drop stream to ignore all data that comes in, + // instead of buffering it indefinitely + drop(m.take_stream()); + Resp::new(EndpointHandler::handle(self, m.msg(), from).await) + } +} + +// ---- + +/// This struct represents an endpoint for message of type `M`. +/// +/// Creating a new endpoint is done by calling `NetApp::endpoint`. +/// An endpoint is identified primarily by its path, which is specified +/// at creation time. +/// +/// An `Endpoint` is used both to send requests to remote nodes, +/// and to specify the handler for such requests on the local node. +/// The type `H` represents the type of the handler object for +/// endpoint messages (see `StreamingEndpointHandler`). +pub struct Endpoint +where + M: Message, + H: StreamingEndpointHandler, +{ + _phantom: PhantomData, + netapp: Arc, + path: String, + handler: ArcSwapOption, +} + +impl Endpoint +where + M: Message, + H: StreamingEndpointHandler, +{ + pub(crate) fn new(netapp: Arc, path: String) -> Self { + Self { + _phantom: PhantomData::default(), + netapp, + path, + handler: ArcSwapOption::from(None), + } + } + + /// Get the path of this endpoint + pub fn path(&self) -> &str { + &self.path + } + + /// Set the object that is responsible of handling requests to + /// this endpoint on the local node. + pub fn set_handler(&self, h: Arc) { + self.handler.swap(Some(h)); + } + + /// Call this endpoint on a remote node (or on the local node, + /// for that matter). This function invokes the full version that + /// allows to attach a stream to the request and to + /// receive such a stream attached to the response. + pub async fn call_streaming( + &self, + target: &NodeID, + req: T, + prio: RequestPriority, + ) -> Result, Error> + where + T: IntoReq, + { + if *target == self.netapp.id { + match self.handler.load_full() { + None => Err(Error::NoHandler), + Some(h) => Ok(h.handle(req.into_req_local(), self.netapp.id).await), + } + } else { + let conn = self + .netapp + .client_conns + .read() + .unwrap() + .get(target) + .cloned(); + match conn { + None => Err(Error::Message(format!( + "Not connected: {}", + hex::encode(&target[..8]) + ))), + Some(c) => c.call(req.into_req()?, self.path.as_str(), prio).await, + } + } + } + + /// Call this endpoint on a remote node. This function is the simplified + /// version that doesn't allow to have streams attached to the request + /// or the response; see `call_streaming` for the full version. + pub async fn call( + &self, + target: &NodeID, + req: M, + prio: RequestPriority, + ) -> Result<::Response, Error> { + Ok(self.call_streaming(target, req, prio).await?.into_msg()) + } +} + +// ---- Internal stuff ---- + +pub(crate) type DynEndpoint = Box; + +pub(crate) trait GenericEndpoint { + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result>; + fn drop_handler(&self); + fn clone_endpoint(&self) -> DynEndpoint; +} + +#[derive(Clone)] +pub(crate) struct EndpointArc(pub(crate) Arc>) +where + M: Message, + H: StreamingEndpointHandler; + +impl GenericEndpoint for EndpointArc +where + M: Message, + H: StreamingEndpointHandler + 'static, +{ + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result> { + async move { + match self.0.handler.load_full() { + None => Err(Error::NoHandler), + Some(h) => { + let req = Req::from_enc(req_enc)?; + let res = h.handle(req, from).await; + Ok(res.into_enc()?) + } + } + } + .boxed() + } + + fn drop_handler(&self) { + self.0.handler.swap(None); + } + + fn clone_endpoint(&self) -> DynEndpoint { + Box::new(Self(self.0.clone())) + } +} diff --git a/src/net/error.rs b/src/net/error.rs new file mode 100644 index 00000000..f67794ed --- /dev/null +++ b/src/net/error.rs @@ -0,0 +1,126 @@ +use std::io; + +use log::error; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("IO error: {0}")] + Io(#[from] io::Error), + + #[error("Messagepack encode error: {0}")] + RMPEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RMPDecode(#[from] rmp_serde::decode::Error), + + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), + + #[error("oneshot receive error: {0}")] + OneshotRecv(#[from] tokio::sync::oneshot::error::RecvError), + + #[error("Handshake error: {0}")] + Handshake(#[from] kuska_handshake::async_std::Error), + + #[error("UTF8 error: {0}")] + UTF8(#[from] std::string::FromUtf8Error), + + #[error("Framing protocol error")] + Framing, + + #[error("Remote error ({0:?}): {1}")] + Remote(io::ErrorKind, String), + + #[error("Request ID collision")] + IdCollision, + + #[error("{0}")] + Message(String), + + #[error("No handler / shutting down")] + NoHandler, + + #[error("Connection closed")] + ConnectionClosed, + + #[error("Version mismatch: {0}")] + VersionMismatch(String), +} + +impl From> for Error { + fn from(_e: tokio::sync::watch::error::SendError) -> Error { + Error::Message("Watch send error".into()) + } +} + +impl From> for Error { + fn from(_e: tokio::sync::mpsc::error::SendError) -> Error { + Error::Message("MPSC send error".into()) + } +} + +/// The trait adds a `.log_err()` method on `Result<(), E>` types, +/// which dismisses the error by logging it to stderr. +pub trait LogError { + fn log_err(self, msg: &'static str); +} + +impl LogError for Result<(), E> +where + E: Into, +{ + fn log_err(self, msg: &'static str) { + if let Err(e) = self { + error!("Error: {}: {}", msg, Into::::into(e)); + }; + } +} + +impl LogError for Result +where + T: LogError, + E: Into, +{ + fn log_err(self, msg: &'static str) { + match self { + Err(e) => error!("Error: {}: {}", msg, Into::::into(e)), + Ok(x) => x.log_err(msg), + } + } +} + +// ---- Helpers for serializing I/O Errors + +pub(crate) fn u8_to_io_errorkind(v: u8) -> std::io::ErrorKind { + use std::io::ErrorKind; + match v { + 101 => ErrorKind::ConnectionAborted, + 102 => ErrorKind::BrokenPipe, + 103 => ErrorKind::WouldBlock, + 104 => ErrorKind::InvalidInput, + 105 => ErrorKind::InvalidData, + 106 => ErrorKind::TimedOut, + 107 => ErrorKind::Interrupted, + 108 => ErrorKind::UnexpectedEof, + 109 => ErrorKind::OutOfMemory, + 110 => ErrorKind::ConnectionReset, + _ => ErrorKind::Other, + } +} + +pub(crate) fn io_errorkind_to_u8(kind: std::io::ErrorKind) -> u8 { + use std::io::ErrorKind; + match kind { + ErrorKind::ConnectionAborted => 101, + ErrorKind::BrokenPipe => 102, + ErrorKind::WouldBlock => 103, + ErrorKind::InvalidInput => 104, + ErrorKind::InvalidData => 105, + ErrorKind::TimedOut => 106, + ErrorKind::Interrupted => 107, + ErrorKind::UnexpectedEof => 108, + ErrorKind::OutOfMemory => 109, + ErrorKind::ConnectionReset => 110, + _ => 100, + } +} diff --git a/src/net/lib.rs b/src/net/lib.rs new file mode 100644 index 00000000..8e30e40f --- /dev/null +++ b/src/net/lib.rs @@ -0,0 +1,35 @@ +//! Netapp is a Rust library that takes care of a few common tasks in distributed software: +//! +//! - establishing secure connections +//! - managing connection lifetime, reconnecting on failure +//! - checking peer's state +//! - peer discovery +//! - query/response message passing model for communications +//! - multiplexing transfers over a connection +//! - overlay networks: full mesh, and soon other methods +//! +//! Of particular interest, read the documentation for the `netapp::NetApp` type, +//! the `message::Message` trait, and `proto::RequestPriority` to learn more +//! about message priorization. +//! Also check out the examples to learn how to use this crate. + +pub mod bytes_buf; +pub mod error; +pub mod stream; +pub mod util; + +pub mod endpoint; +pub mod message; + +mod client; +mod recv; +mod send; +mod server; + +pub mod netapp; +pub mod peering; + +pub use crate::netapp::*; + +#[cfg(test)] +mod test; diff --git a/src/net/message.rs b/src/net/message.rs new file mode 100644 index 00000000..59afb058 --- /dev/null +++ b/src/net/message.rs @@ -0,0 +1,540 @@ +use std::fmt; +use std::marker::PhantomData; +use std::sync::Arc; + +use bytes::{BufMut, Bytes, BytesMut}; +use rand::prelude::*; +use serde::{Deserialize, Serialize}; + +use futures::stream::StreamExt; + +use crate::error::*; +use crate::stream::*; +use crate::util::*; + +/// Priority of a request (click to read more about priorities). +/// +/// This priority value is used to priorize messages +/// in the send queue of the client, and their responses in the send queue of the +/// server. Lower values mean higher priority. +/// +/// This mechanism is useful for messages bigger than the maximum chunk size +/// (set at `0x4000` bytes), such as large file transfers. +/// In such case, all of the messages in the send queue with the highest priority +/// will take turns to send individual chunks, in a round-robin fashion. +/// Once all highest priority messages are sent successfully, the messages with +/// the next highest priority will begin being sent in the same way. +/// +/// The same priority value is given to a request and to its associated response. +pub type RequestPriority = u8; + +// Usage of priority levels in Garage: +// +// PRIO_HIGH +// for liveness check events such as pings and important +// reconfiguration events such as layout changes +// +// PRIO_NORMAL +// for standard interactive requests to exchange metadata +// +// PRIO_NORMAL | PRIO_SECONDARY +// for standard interactive requests to exchange block data +// +// PRIO_BACKGROUND +// for background resync requests to exchange metadata +// PRIO_BACKGROUND | PRIO_SECONDARY +// for background resync requests to exchange block data + +/// Priority class: high +pub const PRIO_HIGH: RequestPriority = 0x20; +/// Priority class: normal +pub const PRIO_NORMAL: RequestPriority = 0x40; +/// Priority class: background +pub const PRIO_BACKGROUND: RequestPriority = 0x80; + +/// Priority: primary among given class +pub const PRIO_PRIMARY: RequestPriority = 0x00; +/// Priority: secondary among given class (ex: `PRIO_HIGH | PRIO_SECONDARY`) +pub const PRIO_SECONDARY: RequestPriority = 0x01; + +// ---- + +/// An order tag can be added to a message or a response to indicate +/// whether it should be sent after or before other messages with order tags +/// referencing a same stream +#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +pub struct OrderTag(pub(crate) u64, pub(crate) u64); + +/// A stream is an opaque identifier that defines a set of messages +/// or responses that are ordered wrt one another using to order tags. +#[derive(Clone, Copy)] +pub struct OrderTagStream(u64); + +impl OrderTag { + /// Create a new stream from which to generate order tags. Example: + /// ```ignore + /// let stream = OrderTag.stream(); + /// let tag_1 = stream.order(1); + /// let tag_2 = stream.order(2); + /// ``` + pub fn stream() -> OrderTagStream { + OrderTagStream(thread_rng().gen()) + } +} +impl OrderTagStream { + /// Create the order tag for message `order` in this stream + pub fn order(&self, order: u64) -> OrderTag { + OrderTag(self.0, order) + } +} + +// ---- + +/// This trait should be implemented by all messages your application +/// wants to handle. It specifies which data type should be sent +/// as a response to this message in the RPC protocol. +pub trait Message: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static { + /// The type of the response that is sent in response to this message + type Response: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static; +} + +// ---- + +/// The Req is a helper object used to create requests and attach them +/// a stream of data. If the stream is a fixed Bytes and not a ByteStream, +/// Req is cheaply cloneable to allow the request to be sent to different +/// peers (Clone will panic if the stream is a ByteStream). +pub struct Req { + pub(crate) msg: Arc, + pub(crate) msg_ser: Option, + pub(crate) stream: AttachedStream, + pub(crate) order_tag: Option, +} + +impl Req { + /// Creates a new request from a base message `M` + pub fn new(v: M) -> Result { + Ok(v.into_req()?) + } + + /// Attach a stream to message in request, where the stream is streamed + /// from a fixed `Bytes` buffer + pub fn with_stream_from_buffer(self, b: Bytes) -> Self { + Self { + stream: AttachedStream::Fixed(b), + ..self + } + } + + /// Attach a stream to message in request, where the stream is + /// an instance of `ByteStream`. Note than when a `Req` has an attached + /// stream which is a `ByteStream` instance, it can no longer be cloned + /// to be sent to different nodes (`.clone()` will panic) + pub fn with_stream(self, b: ByteStream) -> Self { + Self { + stream: AttachedStream::Stream(b), + ..self + } + } + + /// Add an order tag to this request to indicate in which order it should + /// be sent. + pub fn with_order_tag(self, order_tag: OrderTag) -> Self { + Self { + order_tag: Some(order_tag), + ..self + } + } + + /// Get a reference to the message `M` contained in this request + pub fn msg(&self) -> &M { + &self.msg + } + + /// Takes out the stream attached to this request, if any + pub fn take_stream(&mut self) -> Option { + std::mem::replace(&mut self.stream, AttachedStream::None).into_stream() + } + + pub(crate) fn into_enc( + self, + prio: RequestPriority, + path: Bytes, + telemetry_id: Bytes, + ) -> ReqEnc { + ReqEnc { + prio, + path, + telemetry_id, + msg: self.msg_ser.unwrap(), + stream: self.stream.into_stream(), + order_tag: self.order_tag, + } + } + + pub(crate) fn from_enc(enc: ReqEnc) -> Result { + let msg = rmp_serde::decode::from_slice(&enc.msg)?; + Ok(Req { + msg: Arc::new(msg), + msg_ser: Some(enc.msg), + stream: enc + .stream + .map(AttachedStream::Stream) + .unwrap_or(AttachedStream::None), + order_tag: enc.order_tag, + }) + } +} + +/// `IntoReq` represents any object that can be transformed into `Req` +pub trait IntoReq { + /// Transform the object into a `Req`, serializing the message M + /// to be sent to remote nodes + fn into_req(self) -> Result, rmp_serde::encode::Error>; + /// Transform the object into a `Req`, skipping the serialization + /// of message M, in the case we are not sending this RPC message to + /// a remote node + fn into_req_local(self) -> Req; +} + +impl IntoReq for M { + fn into_req(self) -> Result, rmp_serde::encode::Error> { + let msg_ser = rmp_to_vec_all_named(&self)?; + Ok(Req { + msg: Arc::new(self), + msg_ser: Some(Bytes::from(msg_ser)), + stream: AttachedStream::None, + order_tag: None, + }) + } + fn into_req_local(self) -> Req { + Req { + msg: Arc::new(self), + msg_ser: None, + stream: AttachedStream::None, + order_tag: None, + } + } +} + +impl IntoReq for Req { + fn into_req(self) -> Result, rmp_serde::encode::Error> { + Ok(self) + } + fn into_req_local(self) -> Req { + self + } +} + +impl Clone for Req { + fn clone(&self) -> Self { + let stream = match &self.stream { + AttachedStream::None => AttachedStream::None, + AttachedStream::Fixed(b) => AttachedStream::Fixed(b.clone()), + AttachedStream::Stream(_) => { + panic!("Cannot clone a Req<_> with a non-buffer attached stream") + } + }; + Self { + msg: self.msg.clone(), + msg_ser: self.msg_ser.clone(), + stream, + order_tag: self.order_tag, + } + } +} + +impl fmt::Debug for Req +where + M: Message + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "Req[{:?}", self.msg)?; + match &self.stream { + AttachedStream::None => write!(f, "]"), + AttachedStream::Fixed(b) => write!(f, "; stream=buf:{}]", b.len()), + AttachedStream::Stream(_) => write!(f, "; stream]"), + } + } +} + +// ---- + +/// The Resp represents a full response from a RPC that may have +/// an attached stream. +pub struct Resp { + pub(crate) _phantom: PhantomData, + pub(crate) msg: M::Response, + pub(crate) stream: AttachedStream, + pub(crate) order_tag: Option, +} + +impl Resp { + /// Creates a new response from a base response message + pub fn new(v: M::Response) -> Self { + Resp { + _phantom: Default::default(), + msg: v, + stream: AttachedStream::None, + order_tag: None, + } + } + + /// Attach a stream to message in response, where the stream is streamed + /// from a fixed `Bytes` buffer + pub fn with_stream_from_buffer(self, b: Bytes) -> Self { + Self { + stream: AttachedStream::Fixed(b), + ..self + } + } + + /// Attach a stream to message in response, where the stream is + /// an instance of `ByteStream`. + pub fn with_stream(self, b: ByteStream) -> Self { + Self { + stream: AttachedStream::Stream(b), + ..self + } + } + + /// Add an order tag to this response to indicate in which order it should + /// be sent. + pub fn with_order_tag(self, order_tag: OrderTag) -> Self { + Self { + order_tag: Some(order_tag), + ..self + } + } + + /// Get a reference to the response message contained in this request + pub fn msg(&self) -> &M::Response { + &self.msg + } + + /// Transforms the `Resp` into the response message it contains, + /// dropping everything else (including attached data stream) + pub fn into_msg(self) -> M::Response { + self.msg + } + + /// Transforms the `Resp` into, on the one side, the response message + /// it contains, and on the other side, the associated data stream + /// if it exists + pub fn into_parts(self) -> (M::Response, Option) { + (self.msg, self.stream.into_stream()) + } + + pub(crate) fn into_enc(self) -> Result { + Ok(RespEnc { + msg: rmp_to_vec_all_named(&self.msg)?.into(), + stream: self.stream.into_stream(), + order_tag: self.order_tag, + }) + } + + pub(crate) fn from_enc(enc: RespEnc) -> Result { + let msg = rmp_serde::decode::from_slice(&enc.msg)?; + Ok(Self { + _phantom: Default::default(), + msg, + stream: enc + .stream + .map(AttachedStream::Stream) + .unwrap_or(AttachedStream::None), + order_tag: enc.order_tag, + }) + } +} + +impl fmt::Debug for Resp +where + M: Message, + ::Response: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "Resp[{:?}", self.msg)?; + match &self.stream { + AttachedStream::None => write!(f, "]"), + AttachedStream::Fixed(b) => write!(f, "; stream=buf:{}]", b.len()), + AttachedStream::Stream(_) => write!(f, "; stream]"), + } + } +} + +// ---- + +pub(crate) enum AttachedStream { + None, + Fixed(Bytes), + Stream(ByteStream), +} + +impl AttachedStream { + pub fn into_stream(self) -> Option { + match self { + AttachedStream::None => None, + AttachedStream::Fixed(b) => Some(Box::pin(futures::stream::once(async move { Ok(b) }))), + AttachedStream::Stream(s) => Some(s), + } + } +} + +// ---- ---- + +/// Encoding for requests into a ByteStream: +/// - priority: u8 +/// - path length: u8 +/// - path: [u8; path length] +/// - telemetry id length: u8 +/// - telemetry id: [u8; telemetry id length] +/// - msg len: u32 +/// - msg [u8; ..] +/// - the attached stream as the rest of the encoded stream +pub(crate) struct ReqEnc { + pub(crate) prio: RequestPriority, + pub(crate) path: Bytes, + pub(crate) telemetry_id: Bytes, + pub(crate) msg: Bytes, + pub(crate) stream: Option, + pub(crate) order_tag: Option, +} + +impl ReqEnc { + pub(crate) fn encode(self) -> (ByteStream, Option) { + let mut buf = BytesMut::with_capacity( + self.path.len() + self.telemetry_id.len() + self.msg.len() + 16, + ); + + buf.put_u8(self.prio); + + buf.put_u8(self.path.len() as u8); + buf.put(self.path); + + buf.put_u8(self.telemetry_id.len() as u8); + buf.put(&self.telemetry_id[..]); + + buf.put_u32(self.msg.len() as u32); + + let header = buf.freeze(); + + let res_stream: ByteStream = if let Some(stream) = self.stream { + Box::pin(futures::stream::iter([Ok(header), Ok(self.msg)]).chain(stream)) + } else { + Box::pin(futures::stream::iter([Ok(header), Ok(self.msg)])) + }; + (res_stream, self.order_tag) + } + + pub(crate) async fn decode(stream: ByteStream) -> Result { + Self::decode_aux(stream) + .await + .map_err(read_exact_error_to_error) + } + + async fn decode_aux(stream: ByteStream) -> Result { + let mut reader = ByteStreamReader::new(stream); + + let prio = reader.read_u8().await?; + + let path_len = reader.read_u8().await?; + let path = reader.read_exact(path_len as usize).await?; + + let telemetry_id_len = reader.read_u8().await?; + let telemetry_id = reader.read_exact(telemetry_id_len as usize).await?; + + let msg_len = reader.read_u32().await?; + let msg = reader.read_exact(msg_len as usize).await?; + + Ok(Self { + prio, + path, + telemetry_id, + msg, + stream: Some(reader.into_stream()), + order_tag: None, + }) + } +} + +/// Encoding for responses into a ByteStream: +/// IF SUCCESS: +/// - 0: u8 +/// - msg len: u32 +/// - msg [u8; ..] +/// - the attached stream as the rest of the encoded stream +/// IF ERROR: +/// - message length + 1: u8 +/// - error code: u8 +/// - message: [u8; message_length] +pub(crate) struct RespEnc { + msg: Bytes, + stream: Option, + order_tag: Option, +} + +impl RespEnc { + pub(crate) fn encode(resp: Result) -> (ByteStream, Option) { + match resp { + Ok(Self { + msg, + stream, + order_tag, + }) => { + let mut buf = BytesMut::with_capacity(4); + buf.put_u32(msg.len() as u32); + let header = buf.freeze(); + + let res_stream: ByteStream = if let Some(stream) = stream { + Box::pin(futures::stream::iter([Ok(header), Ok(msg)]).chain(stream)) + } else { + Box::pin(futures::stream::iter([Ok(header), Ok(msg)])) + }; + (res_stream, order_tag) + } + Err(err) => { + let err = std::io::Error::new( + std::io::ErrorKind::Other, + format!("netapp error: {}", err), + ); + ( + Box::pin(futures::stream::once(async move { Err(err) })), + None, + ) + } + } + } + + pub(crate) async fn decode(stream: ByteStream) -> Result { + Self::decode_aux(stream) + .await + .map_err(read_exact_error_to_error) + } + + async fn decode_aux(stream: ByteStream) -> Result { + let mut reader = ByteStreamReader::new(stream); + + let msg_len = reader.read_u32().await?; + let msg = reader.read_exact(msg_len as usize).await?; + + // Check whether the response stream still has data or not. + // If no more data is coming, this will defuse the request canceller. + // If we didn't do this, and the client doesn't try to read from the stream, + // the request canceller doesn't know that we read everything and + // sends a cancellation message to the server (which they don't care about). + reader.fill_buffer().await; + + Ok(Self { + msg, + stream: Some(reader.into_stream()), + order_tag: None, + }) + } +} + +fn read_exact_error_to_error(e: ReadExactError) -> Error { + match e { + ReadExactError::Stream(err) => Error::Remote(err.kind(), err.to_string()), + ReadExactError::UnexpectedEos => Error::Framing, + } +} diff --git a/src/net/netapp.rs b/src/net/netapp.rs new file mode 100644 index 00000000..36c6fc88 --- /dev/null +++ b/src/net/netapp.rs @@ -0,0 +1,470 @@ +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, RwLock}; + +use log::{debug, error, info, trace, warn}; + +use arc_swap::ArcSwapOption; + +use serde::{Deserialize, Serialize}; +use sodiumoxide::crypto::auth; +use sodiumoxide::crypto::sign::ed25519; + +use futures::stream::futures_unordered::FuturesUnordered; +use futures::stream::StreamExt; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::select; +use tokio::sync::{mpsc, watch}; + +use crate::client::*; +use crate::endpoint::*; +use crate::error::*; +use crate::message::*; +use crate::server::*; + +/// A node's identifier, which is also its public cryptographic key +pub type NodeID = sodiumoxide::crypto::sign::ed25519::PublicKey; +/// A node's secret key +pub type NodeKey = sodiumoxide::crypto::sign::ed25519::SecretKey; +/// A network key +pub type NetworkKey = sodiumoxide::crypto::auth::Key; + +/// Tag which is exchanged between client and server upon connection establishment +/// to check that they are running compatible versions of Netapp, +/// composed of 8 bytes for Netapp version and 8 bytes for client version +pub(crate) type VersionTag = [u8; 16]; + +/// Value of garage_net version used in the version tag +/// We are no longer using prefix `netapp` as garage_net is forked from the netapp crate. +/// Since Garage v1.0, we have replaced the prefix by `grgnet` (shorthand for garage_net). +pub(crate) const NETAPP_VERSION_TAG: u64 = 0x6772676e65740010; // grgnet 0x0010 (1.0) + +/// HelloMessage is sent by the client on a Netapp connection to indicate +/// that they are also a server and ready to receive incoming connections +/// at the specified address and port. If the client doesn't know their +/// public address, they don't need to specify it and we look at the +/// remote address of the socket is used instead. +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct HelloMessage { + pub server_addr: Option, + pub server_port: u16, +} + +impl Message for HelloMessage { + type Response = (); +} + +type OnConnectHandler = Box; +type OnDisconnectHandler = Box; + +/// NetApp is the main class that handles incoming and outgoing connections. +/// +/// NetApp can be used in a stand-alone fashion or together with a peering strategy. +/// If using it alone, you will want to set `on_connect` and `on_disconnect` events +/// in order to manage information about the current peer list. +pub struct NetApp { + bind_outgoing_to: Option, + listen_params: ArcSwapOption, + + /// Version tag, 8 bytes for netapp version, 8 bytes for app version + pub version_tag: VersionTag, + /// Network secret key + pub netid: auth::Key, + /// Our peer ID + pub id: NodeID, + /// Private key associated with our peer ID + pub privkey: ed25519::SecretKey, + + pub(crate) server_conns: RwLock>>, + pub(crate) client_conns: RwLock>>, + + pub(crate) endpoints: RwLock>, + hello_endpoint: ArcSwapOption>, + + on_connected_handler: ArcSwapOption, + on_disconnected_handler: ArcSwapOption, +} + +struct ListenParams { + listen_addr: SocketAddr, + public_addr: Option, +} + +impl NetApp { + /// Creates a new instance of NetApp, which can serve either as a full p2p node, + /// or just as a passive client. To upgrade to a full p2p node, spawn a listener + /// using `.listen()` + /// + /// Our Peer ID is the public key associated to the secret key given here. + pub fn new( + app_version_tag: u64, + netid: auth::Key, + privkey: ed25519::SecretKey, + bind_outgoing_to: Option, + ) -> Arc { + let mut version_tag = [0u8; 16]; + version_tag[0..8].copy_from_slice(&u64::to_be_bytes(NETAPP_VERSION_TAG)[..]); + version_tag[8..16].copy_from_slice(&u64::to_be_bytes(app_version_tag)[..]); + + let id = privkey.public_key(); + let netapp = Arc::new(Self { + bind_outgoing_to, + listen_params: ArcSwapOption::new(None), + version_tag, + netid, + id, + privkey, + server_conns: RwLock::new(HashMap::new()), + client_conns: RwLock::new(HashMap::new()), + endpoints: RwLock::new(HashMap::new()), + hello_endpoint: ArcSwapOption::new(None), + on_connected_handler: ArcSwapOption::new(None), + on_disconnected_handler: ArcSwapOption::new(None), + }); + + netapp + .hello_endpoint + .swap(Some(netapp.endpoint("garage_net/netapp.rs/Hello".into()))); + netapp + .hello_endpoint + .load_full() + .unwrap() + .set_handler(netapp.clone()); + + netapp + } + + /// Set the handler to be called when a new connection (incoming or outgoing) has + /// been successfully established. Do not set this if using a peering strategy, + /// as the peering strategy will need to set this itself. + pub fn on_connected(&self, handler: F) + where + F: Fn(NodeID, SocketAddr, bool) + Sized + Send + Sync + 'static, + { + self.on_connected_handler + .store(Some(Arc::new(Box::new(handler)))); + } + + /// Set the handler to be called when an existing connection (incoming or outgoing) has + /// been closed by either party. Do not set this if using a peering strategy, + /// as the peering strategy will need to set this itself. + pub fn on_disconnected(&self, handler: F) + where + F: Fn(NodeID, bool) + Sized + Send + Sync + 'static, + { + self.on_disconnected_handler + .store(Some(Arc::new(Box::new(handler)))); + } + + /// Create a new endpoint with path `path`, + /// that handles messages of type `M`. + /// `H` is the type of the object that should handle requests + /// to this endpoint on the local node. If you don't want + /// to handle request on the local node (e.g. if this node + /// is only a client in the network), define the type `H` + /// to be `()`. + /// This function will panic if the endpoint has already been + /// created. + pub fn endpoint(self: &Arc, path: String) -> Arc> + where + M: Message + 'static, + H: StreamingEndpointHandler + 'static, + { + let endpoint = Arc::new(Endpoint::::new(self.clone(), path.clone())); + let endpoint_arc = EndpointArc(endpoint.clone()); + if self + .endpoints + .write() + .unwrap() + .insert(path.clone(), Box::new(endpoint_arc)) + .is_some() + { + panic!("Redefining endpoint: {}", path); + }; + endpoint + } + + /// Main listening process for our app. This future runs during the whole + /// run time of our application. + /// If this is not called, the NetApp instance remains a passive client. + pub async fn listen( + self: Arc, + listen_addr: SocketAddr, + public_addr: Option, + mut must_exit: watch::Receiver, + ) { + let listen_params = ListenParams { + listen_addr, + public_addr, + }; + if self + .listen_params + .swap(Some(Arc::new(listen_params))) + .is_some() + { + error!("Trying to listen on NetApp but we're already listening!"); + } + + let listener = TcpListener::bind(listen_addr).await.unwrap(); + info!("Listening on {}", listen_addr); + + let (conn_in, mut conn_out) = mpsc::unbounded_channel(); + let connection_collector = tokio::spawn(async move { + let mut collection = FuturesUnordered::new(); + loop { + if collection.is_empty() { + match conn_out.recv().await { + Some(f) => collection.push(f), + None => break, + } + } else { + select! { + new_fut = conn_out.recv() => { + match new_fut { + Some(f) => collection.push(f), + None => break, + } + } + result = collection.next() => { + trace!("Collected connection: {:?}", result); + } + } + } + } + debug!("Collecting last open server connections."); + while let Some(conn_res) = collection.next().await { + trace!("Collected connection: {:?}", conn_res); + } + debug!("No more server connections to collect"); + }); + + while !*must_exit.borrow_and_update() { + let (socket, peer_addr) = select! { + sockres = listener.accept() => { + match sockres { + Ok(x) => x, + Err(e) => { + warn!("Error in listener.accept: {}", e); + continue; + } + } + }, + _ = must_exit.changed() => continue, + }; + + info!( + "Incoming connection from {}, negotiating handshake...", + peer_addr + ); + let self2 = self.clone(); + let must_exit2 = must_exit.clone(); + conn_in + .send(tokio::spawn(async move { + ServerConn::run(self2, socket, must_exit2) + .await + .log_err("ServerConn::run"); + })) + .log_err("Failed to send connection to connection collector"); + } + + drop(conn_in); + + connection_collector + .await + .log_err("Failed to await for connection collector"); + } + + /// Drop all endpoint handlers, as well as handlers for connection/disconnection + /// events. (This disables the peering strategy) + /// + /// Use this when terminating to break reference cycles + pub fn drop_all_handlers(&self) { + for (_, endpoint) in self.endpoints.read().unwrap().iter() { + endpoint.drop_handler(); + } + self.on_connected_handler.store(None); + self.on_disconnected_handler.store(None); + } + + /// Attempt to connect to a peer, given by its ip:port and its public key. + /// The public key will be checked during the secret handshake process. + /// This function returns once the connection has been established and a + /// successful handshake was made. At this point we can send messages to + /// the other node with `Netapp::request` + pub async fn try_connect(self: Arc, ip: SocketAddr, id: NodeID) -> Result<(), Error> { + // Don't connect to ourself, we don't care + if id == self.id { + return Ok(()); + } + + // Don't connect if already connected + if self.client_conns.read().unwrap().contains_key(&id) { + return Ok(()); + } + + let stream = match self.bind_outgoing_to { + Some(addr) => { + let socket = if addr.is_ipv4() { + TcpSocket::new_v4()? + } else { + TcpSocket::new_v6()? + }; + socket.bind(SocketAddr::new(addr, 0))?; + socket.connect(ip).await? + } + None => TcpStream::connect(ip).await?, + }; + info!("Connected to {}, negotiating handshake...", ip); + ClientConn::init(self, stream, id).await?; + Ok(()) + } + + /// Close the outgoing connection we have to a node specified by its public key, + /// if such a connection is currently open. + pub fn disconnect(self: &Arc, id: &NodeID) { + let conn = self.client_conns.write().unwrap().remove(id); + + // If id is ourself, we're not supposed to have a connection open + if *id == self.id { + // sanity check + assert!(conn.is_none(), "had a connection to local node"); + return; + } + + if let Some(c) = conn { + debug!( + "Closing connection to {} ({})", + hex::encode(&c.peer_id[..8]), + c.remote_addr + ); + c.close(); + + // call on_disconnected_handler immediately, since the connection was removed + let id = *id; + let self2 = self.clone(); + tokio::spawn(async move { + if let Some(h) = self2.on_disconnected_handler.load().as_ref() { + h(id, false); + } + }); + } + } + + // Called from conn.rs when an incoming connection is successfully established + // Registers the connection in our list of connections + // Do not yet call the on_connected handler, because we don't know if the remote + // has an actual IP address and port we can call them back on. + // We will know this when they send a Hello message, which is handled below. + pub(crate) fn connected_as_server(&self, id: NodeID, conn: Arc) { + info!( + "Accepted connection from {} at {}", + hex::encode(&id[..8]), + conn.remote_addr + ); + + self.server_conns.write().unwrap().insert(id, conn); + } + + // Handle hello message from a client. This message is used for them to tell us + // that they are listening on a certain port number on which we can call them back. + // At this point we know they are a full network member, and not just a client, + // and we call the on_connected handler so that the peering strategy knows + // we have a new potential peer + + // Called from conn.rs when an incoming connection is closed. + // We deregister the connection from server_conns and call the + // handler registered by on_disconnected + pub(crate) fn disconnected_as_server(&self, id: &NodeID, conn: Arc) { + info!("Connection from {} closed", hex::encode(&id[..8])); + + let mut conn_list = self.server_conns.write().unwrap(); + if let Some(c) = conn_list.get(id) { + if Arc::ptr_eq(c, &conn) { + conn_list.remove(id); + drop(conn_list); + + if let Some(h) = self.on_disconnected_handler.load().as_ref() { + h(conn.peer_id, true); + } + } + } + } + + // Called from conn.rs when an outgoinc connection is successfully established. + // The connection is registered in self.client_conns, and the + // on_connected handler is called. + // + // Since we are ourself listening, we send them a Hello message so that + // they know on which port to call us back. (TODO: don't do this if we are + // just a simple client and not a full p2p node) + pub(crate) fn connected_as_client(&self, id: NodeID, conn: Arc) { + info!("Connection established to {}", hex::encode(&id[..8])); + + { + let old_c_opt = self.client_conns.write().unwrap().insert(id, conn.clone()); + if let Some(old_c) = old_c_opt { + tokio::spawn(async move { old_c.close() }); + } + } + + if let Some(h) = self.on_connected_handler.load().as_ref() { + h(conn.peer_id, conn.remote_addr, false); + } + + if let Some(lp) = self.listen_params.load_full() { + let server_addr = lp.public_addr.map(|x| x.ip()); + let server_port = lp + .public_addr + .map(|x| x.port()) + .unwrap_or(lp.listen_addr.port()); + let hello_endpoint = self.hello_endpoint.load_full().unwrap(); + tokio::spawn(async move { + hello_endpoint + .call( + &conn.peer_id, + HelloMessage { + server_addr, + server_port, + }, + PRIO_NORMAL, + ) + .await + .map(|_| ()) + .log_err("Sending hello message"); + }); + } + } + + // Called from conn.rs when an outgoinc connection is closed. + // The connection is removed from conn_list, and the on_disconnected handler + // is called. + pub(crate) fn disconnected_as_client(&self, id: &NodeID, conn: Arc) { + info!("Connection to {} closed", hex::encode(&id[..8])); + let mut conn_list = self.client_conns.write().unwrap(); + if let Some(c) = conn_list.get(id) { + if Arc::ptr_eq(c, &conn) { + conn_list.remove(id); + drop(conn_list); + + if let Some(h) = self.on_disconnected_handler.load().as_ref() { + h(conn.peer_id, false); + } + } + } + // else case: happens if connection was removed in .disconnect() + // in which case on_disconnected_handler was already called + } +} + +impl EndpointHandler for NetApp { + async fn handle(self: &Arc, msg: &HelloMessage, from: NodeID) { + debug!("Hello from {:?}: {:?}", hex::encode(&from[..8]), msg); + if let Some(h) = self.on_connected_handler.load().as_ref() { + if let Some(c) = self.server_conns.read().unwrap().get(&from) { + let remote_ip = msg.server_addr.unwrap_or_else(|| c.remote_addr.ip()); + let remote_addr = SocketAddr::new(remote_ip, msg.server_port); + h(from, remote_addr, true); + } + } + } +} diff --git a/src/net/peering.rs b/src/net/peering.rs new file mode 100644 index 00000000..08378a08 --- /dev/null +++ b/src/net/peering.rs @@ -0,0 +1,615 @@ +use std::collections::{HashMap, VecDeque}; +use std::net::SocketAddr; +use std::sync::atomic::{self, AtomicU64}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwap; +use log::{debug, info, trace, warn}; +use serde::{Deserialize, Serialize}; + +use tokio::select; +use tokio::sync::watch; + +use sodiumoxide::crypto::hash; + +use crate::endpoint::*; +use crate::error::*; +use crate::netapp::*; + +use crate::message::*; +use crate::NodeID; + +const CONN_RETRY_INTERVAL: Duration = Duration::from_secs(30); +const CONN_MAX_RETRIES: usize = 10; +const PING_INTERVAL: Duration = Duration::from_secs(15); +const LOOP_DELAY: Duration = Duration::from_secs(1); +const FAILED_PING_THRESHOLD: usize = 4; + +const DEFAULT_PING_TIMEOUT_MILLIS: u64 = 10_000; + +// -- Protocol messages -- + +#[derive(Serialize, Deserialize)] +struct PingMessage { + pub id: u64, + pub peer_list_hash: hash::Digest, +} + +impl Message for PingMessage { + type Response = PingMessage; +} + +#[derive(Serialize, Deserialize)] +struct PeerListMessage { + pub list: Vec<(NodeID, SocketAddr)>, +} + +impl Message for PeerListMessage { + type Response = PeerListMessage; +} + +// -- Algorithm data structures -- + +#[derive(Debug)] +struct PeerInfoInternal { + // known_addrs contains all of the addresses everyone gave us + known_addrs: Vec, + + state: PeerConnState, + last_send_ping: Option, + last_seen: Option, + ping: VecDeque, + failed_pings: usize, +} + +impl PeerInfoInternal { + fn new(state: PeerConnState, known_addr: Option) -> Self { + Self { + known_addrs: known_addr.map(|x| vec![x]).unwrap_or_default(), + state, + last_send_ping: None, + last_seen: None, + ping: VecDeque::new(), + failed_pings: 0, + } + } + fn add_addr(&mut self, addr: SocketAddr) -> bool { + if !self.known_addrs.contains(&addr) { + self.known_addrs.push(addr); + // If we are learning a new address for this node, + // we want to retry connecting + self.state = match self.state { + PeerConnState::Trying(_) => PeerConnState::Trying(0), + PeerConnState::Waiting(_, _) | PeerConnState::Abandonned => { + PeerConnState::Waiting(0, Instant::now()) + } + x @ (PeerConnState::Ourself | PeerConnState::Connected { .. }) => x, + }; + true + } else { + false + } + } +} + +/// Information that the full mesh peering strategy can return about the peers it knows of +#[derive(Copy, Clone, Debug)] +pub struct PeerInfo { + /// The node's identifier (its public key) + pub id: NodeID, + /// The current status of our connection to this node + pub state: PeerConnState, + /// The last time at which the node was seen + pub last_seen: Option, + /// The average ping to this node on recent observations (if at least one ping value is known) + pub avg_ping: Option, + /// The maximum observed ping to this node on recent observations (if at least one + /// ping value is known) + pub max_ping: Option, + /// The median ping to this node on recent observations (if at least one ping value + /// is known) + pub med_ping: Option, +} + +impl PeerInfo { + /// Returns true if we can currently send requests to this peer + pub fn is_up(&self) -> bool { + self.state.is_up() + } +} + +/// PeerConnState: possible states for our tentative connections to given peer +/// This structure is only interested in recording connection info for outgoing +/// TCP connections +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum PeerConnState { + /// This entry represents ourself (the local node) + Ourself, + + /// We currently have a connection to this peer + Connected { addr: SocketAddr }, + + /// Our next connection tentative (the nth, where n is the first value of the tuple) + /// will be at given Instant + Waiting(usize, Instant), + + /// A connection tentative is in progress (the nth, where n is the value stored) + Trying(usize), + + /// We abandoned trying to connect to this peer (too many failed attempts) + Abandonned, +} + +impl PeerConnState { + /// Returns true if we can currently send requests to this peer + pub fn is_up(&self) -> bool { + matches!(self, Self::Ourself | Self::Connected { .. }) + } +} + +struct KnownHosts { + list: HashMap, + hash: hash::Digest, +} + +impl KnownHosts { + fn new() -> Self { + let list = HashMap::new(); + let mut ret = Self { + list, + hash: hash::Digest::from_slice(&[0u8; 64][..]).unwrap(), + }; + ret.update_hash(); + ret + } + fn update_hash(&mut self) { + // The hash is a value that is exchanged between nodes when they ping one + // another. Nodes compare their known hosts hash to know if they are connected + // to the same set of nodes. If the hashes differ, they are connected to + // different nodes and they trigger an exchange of the full list of active + // connections. The hash value only represents the set of node IDs and not + // their actual socket addresses, because nodes can be connected via different + // addresses and that shouldn't necessarily trigger a full peer exchange. + let mut list = self + .list + .iter() + .filter(|(_, peer)| peer.state.is_up()) + .map(|(id, _)| *id) + .collect::>(); + list.sort(); + let mut hash_state = hash::State::new(); + for id in list { + hash_state.update(&id[..]); + } + self.hash = hash_state.finalize(); + } + fn connected_peers_vec(&self) -> Vec<(NodeID, SocketAddr)> { + self.list + .iter() + .filter_map(|(id, peer)| match peer.state { + PeerConnState::Connected { addr } => Some((*id, addr)), + _ => None, + }) + .collect::>() + } +} + +/// A "Full Mesh" peering strategy is a peering strategy that tries +/// to establish and maintain a direct connection with all of the +/// known nodes in the network. +pub struct PeeringManager { + netapp: Arc, + known_hosts: RwLock, + public_peer_list: ArcSwap>, + + next_ping_id: AtomicU64, + ping_endpoint: Arc>, + peer_list_endpoint: Arc>, + + ping_timeout_millis: AtomicU64, +} + +impl PeeringManager { + /// Create a new Full Mesh peering strategy. + /// The strategy will not be run until `.run()` is called and awaited. + /// Once that happens, the peering strategy will try to connect + /// to all of the nodes specified in the bootstrap list. + pub fn new( + netapp: Arc, + bootstrap_list: Vec<(NodeID, SocketAddr)>, + our_addr: Option, + ) -> Arc { + let mut known_hosts = KnownHosts::new(); + for (id, addr) in bootstrap_list { + if id != netapp.id { + known_hosts.list.insert( + id, + PeerInfoInternal::new(PeerConnState::Waiting(0, Instant::now()), Some(addr)), + ); + } + } + + known_hosts.list.insert( + netapp.id, + PeerInfoInternal::new(PeerConnState::Ourself, our_addr), + ); + known_hosts.update_hash(); + + let strat = Arc::new(Self { + netapp: netapp.clone(), + known_hosts: RwLock::new(known_hosts), + public_peer_list: ArcSwap::new(Arc::new(Vec::new())), + next_ping_id: AtomicU64::new(42), + ping_endpoint: netapp.endpoint("garage_net/peering.rs/Ping".into()), + peer_list_endpoint: netapp.endpoint("garage_net/peering.rs/PeerList".into()), + ping_timeout_millis: DEFAULT_PING_TIMEOUT_MILLIS.into(), + }); + + strat.update_public_peer_list(&strat.known_hosts.read().unwrap()); + + strat.ping_endpoint.set_handler(strat.clone()); + strat.peer_list_endpoint.set_handler(strat.clone()); + + let strat2 = strat.clone(); + netapp.on_connected(move |id: NodeID, addr: SocketAddr, is_incoming: bool| { + strat2.on_connected(id, addr, is_incoming); + }); + + let strat2 = strat.clone(); + netapp.on_disconnected(move |id: NodeID, is_incoming: bool| { + strat2.on_disconnected(id, is_incoming); + }); + + strat + } + + /// Run the full mesh peering strategy. + /// This future exits when the `must_exit` watch becomes true. + pub async fn run(self: Arc, must_exit: watch::Receiver) { + while !*must_exit.borrow() { + // 1. Read current state: get list of connected peers (ping them) + let (to_ping, to_retry) = { + let known_hosts = self.known_hosts.read().unwrap(); + trace!("known_hosts: {} peers", known_hosts.list.len()); + + let mut to_ping = vec![]; + let mut to_retry = vec![]; + for (id, info) in known_hosts.list.iter() { + trace!("{}, {:?}", hex::encode(&id[..8]), info); + match info.state { + PeerConnState::Connected { .. } => { + let must_ping = match info.last_send_ping { + None => true, + Some(t) => Instant::now() - t > PING_INTERVAL, + }; + if must_ping { + to_ping.push(*id); + } + } + PeerConnState::Waiting(_, t) => { + if Instant::now() >= t { + to_retry.push(*id); + } + } + _ => (), + } + } + (to_ping, to_retry) + }; + + // 2. Dispatch ping to hosts + trace!("to_ping: {} peers", to_ping.len()); + if !to_ping.is_empty() { + let mut known_hosts = self.known_hosts.write().unwrap(); + for id in to_ping.iter() { + known_hosts.list.get_mut(id).unwrap().last_send_ping = Some(Instant::now()); + } + drop(known_hosts); + for id in to_ping { + tokio::spawn(self.clone().ping(id)); + } + } + + // 3. Try reconnects + trace!("to_retry: {} peers", to_retry.len()); + if !to_retry.is_empty() { + let mut known_hosts = self.known_hosts.write().unwrap(); + for id in to_retry { + if let Some(h) = known_hosts.list.get_mut(&id) { + if let PeerConnState::Waiting(i, _) = h.state { + info!( + "Retrying connection to {} at {} ({})", + hex::encode(&id[..8]), + h.known_addrs + .iter() + .map(|x| format!("{}", x)) + .collect::>() + .join(", "), + i + 1 + ); + h.state = PeerConnState::Trying(i); + + let addresses = h.known_addrs.clone(); + tokio::spawn(self.clone().try_connect(id, addresses)); + } + } + } + self.update_public_peer_list(&known_hosts); + } + + // 4. Sleep before next loop iteration + tokio::time::sleep(LOOP_DELAY).await; + } + } + + /// Returns a list of currently known peers in the network. + pub fn get_peer_list(&self) -> Arc> { + self.public_peer_list.load_full() + } + + /// Set the timeout for ping messages, in milliseconds + pub fn set_ping_timeout_millis(&self, timeout: u64) { + self.ping_timeout_millis + .store(timeout, atomic::Ordering::Relaxed); + } + + // -- internal stuff -- + + fn update_public_peer_list(&self, known_hosts: &KnownHosts) { + let mut pub_peer_list = Vec::with_capacity(known_hosts.list.len()); + for (id, info) in known_hosts.list.iter() { + if *id == self.netapp.id { + // sanity check + assert!(matches!(info.state, PeerConnState::Ourself)); + } + let mut pings = info.ping.iter().cloned().collect::>(); + pings.sort(); + if !pings.is_empty() { + pub_peer_list.push(PeerInfo { + id: *id, + state: info.state, + last_seen: info.last_seen, + avg_ping: Some(pings.iter().sum::().div_f64(pings.len() as f64)), + max_ping: pings.last().cloned(), + med_ping: Some(pings[pings.len() / 2]), + }); + } else { + pub_peer_list.push(PeerInfo { + id: *id, + state: info.state, + last_seen: info.last_seen, + avg_ping: None, + max_ping: None, + med_ping: None, + }); + } + } + self.public_peer_list.store(Arc::new(pub_peer_list)); + } + + async fn ping(self: Arc, id: NodeID) { + let peer_list_hash = self.known_hosts.read().unwrap().hash; + let ping_id = self.next_ping_id.fetch_add(1u64, atomic::Ordering::Relaxed); + let ping_time = Instant::now(); + let ping_timeout = + Duration::from_millis(self.ping_timeout_millis.load(atomic::Ordering::Relaxed)); + let ping_msg = PingMessage { + id: ping_id, + peer_list_hash, + }; + + debug!( + "Sending ping {} to {} at {:?}", + ping_id, + hex::encode(&id[..8]), + ping_time + ); + let ping_response = select! { + r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH) => r, + _ = tokio::time::sleep(ping_timeout) => Err(Error::Message("Ping timeout".into())), + }; + + match ping_response { + Err(e) => { + warn!("Error pinging {}: {}", hex::encode(&id[..8]), e); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.failed_pings += 1; + if host.failed_pings > FAILED_PING_THRESHOLD { + warn!( + "Too many failed pings from {}, closing connection.", + hex::encode(&id[..8]) + ); + // this will later update info in known_hosts + // through the disconnection handler + self.netapp.disconnect(&id); + } + } + } + Ok(ping_resp) => { + let resp_time = Instant::now(); + debug!( + "Got ping response from {} at {:?}", + hex::encode(&id[..8]), + resp_time + ); + { + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.failed_pings = 0; + host.last_seen = Some(resp_time); + host.ping.push_back(resp_time - ping_time); + while host.ping.len() > 10 { + host.ping.pop_front(); + } + self.update_public_peer_list(&known_hosts); + } + } + if ping_resp.peer_list_hash != peer_list_hash { + self.exchange_peers(&id).await; + } + } + } + } + + async fn exchange_peers(self: Arc, id: &NodeID) { + let peer_list = self.known_hosts.read().unwrap().connected_peers_vec(); + let pex_message = PeerListMessage { list: peer_list }; + match self + .peer_list_endpoint + .call(id, pex_message, PRIO_BACKGROUND) + .await + { + Err(e) => warn!("Error doing peer exchange: {}", e), + Ok(resp) => { + self.handle_peer_list(&resp.list[..]); + } + } + } + + fn handle_peer_list(&self, list: &[(NodeID, SocketAddr)]) { + let mut known_hosts = self.known_hosts.write().unwrap(); + + let mut changed = false; + for (id, addr) in list.iter() { + if let Some(kh) = known_hosts.list.get_mut(id) { + if kh.add_addr(*addr) { + changed = true; + } + } else { + known_hosts.list.insert(*id, self.new_peer(id, *addr)); + changed = true; + } + } + + if changed { + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + } + + async fn try_connect(self: Arc, id: NodeID, addresses: Vec) { + let conn_addr = { + let mut ret = None; + for addr in addresses.iter() { + debug!("Trying address {} for peer {}", addr, hex::encode(&id[..8])); + match self.netapp.clone().try_connect(*addr, id).await { + Ok(()) => { + ret = Some(*addr); + break; + } + Err(e) => { + debug!( + "Error connecting to {} at {}: {}", + hex::encode(&id[..8]), + addr, + e + ); + } + } + } + ret + }; + + if let Some(ok_addr) = conn_addr { + self.on_connected(id, ok_addr, false); + } else { + warn!( + "Could not connect to peer {} ({} addresses tried)", + hex::encode(&id[..8]), + addresses.len() + ); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = match host.state { + PeerConnState::Trying(i) => { + if i >= CONN_MAX_RETRIES { + PeerConnState::Abandonned + } else { + PeerConnState::Waiting(i + 1, Instant::now() + CONN_RETRY_INTERVAL) + } + } + _ => PeerConnState::Waiting(0, Instant::now() + CONN_RETRY_INTERVAL), + }; + self.update_public_peer_list(&known_hosts); + } + } + } + + fn on_connected(self: &Arc, id: NodeID, addr: SocketAddr, is_incoming: bool) { + if id == self.netapp.id { + // sanity check + panic!( + "on_connected from local node, id={:?}, addr={}, incoming={}", + id, addr, is_incoming + ); + } + + let mut known_hosts = self.known_hosts.write().unwrap(); + if is_incoming { + if let Some(host) = known_hosts.list.get_mut(&id) { + host.add_addr(addr); + } else { + known_hosts.list.insert(id, self.new_peer(&id, addr)); + } + } else { + info!( + "Successfully connected to {} at {}", + hex::encode(&id[..8]), + addr + ); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = PeerConnState::Connected { addr }; + host.add_addr(addr); + } else { + known_hosts.list.insert( + id, + PeerInfoInternal::new(PeerConnState::Connected { addr }, Some(addr)), + ); + } + } + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + + fn on_disconnected(self: &Arc, id: NodeID, is_incoming: bool) { + if !is_incoming { + info!("Connection to {} was closed", hex::encode(&id[..8])); + let mut known_hosts = self.known_hosts.write().unwrap(); + if let Some(host) = known_hosts.list.get_mut(&id) { + host.state = PeerConnState::Waiting(0, Instant::now()); + known_hosts.update_hash(); + self.update_public_peer_list(&known_hosts); + } + } + } + + fn new_peer(&self, id: &NodeID, addr: SocketAddr) -> PeerInfoInternal { + assert!(*id != self.netapp.id); + PeerInfoInternal::new(PeerConnState::Waiting(0, Instant::now()), Some(addr)) + } +} + +impl EndpointHandler for PeeringManager { + async fn handle(self: &Arc, ping: &PingMessage, from: NodeID) -> PingMessage { + let ping_resp = PingMessage { + id: ping.id, + peer_list_hash: self.known_hosts.read().unwrap().hash, + }; + debug!("Ping from {}", hex::encode(&from[..8])); + ping_resp + } +} + +impl EndpointHandler for PeeringManager { + async fn handle( + self: &Arc, + peer_list: &PeerListMessage, + _from: NodeID, + ) -> PeerListMessage { + self.handle_peer_list(&peer_list.list[..]); + let peer_list = self.known_hosts.read().unwrap().connected_peers_vec(); + PeerListMessage { list: peer_list } + } +} diff --git a/src/net/recv.rs b/src/net/recv.rs new file mode 100644 index 00000000..35a6d71a --- /dev/null +++ b/src/net/recv.rs @@ -0,0 +1,151 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use bytes::Bytes; +use log::*; + +use futures::AsyncReadExt; +use tokio::sync::mpsc; + +use crate::error::*; +use crate::send::*; +use crate::stream::*; + +/// Structure to warn when the sender is dropped before end of stream was reached, like when +/// connection to some remote drops while transmitting data +struct Sender { + inner: Option>, +} + +impl Sender { + fn new(inner: mpsc::UnboundedSender) -> Self { + Sender { inner: Some(inner) } + } + + fn send(&self, packet: Packet) { + let _ = self.inner.as_ref().unwrap().send(packet); + } + + fn end(&mut self) { + self.inner = None; + } +} + +impl Drop for Sender { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + let _ = inner.send(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "Netapp connection dropped before end of stream", + ))); + } + } +} + +/// The RecvLoop trait, which is implemented both by the client and the server +/// connection objects (ServerConn and ClientConn) adds a method `.recv_loop()` +/// and a prototype of a handler for received messages `.recv_handler()` that +/// must be filled by implementors. `.recv_loop()` receives messages in a loop +/// according to the protocol defined above: chunks of message in progress of being +/// received are stored in a buffer, and when the last chunk of a message is received, +/// the full message is passed to the receive handler. +pub(crate) trait RecvLoop: Sync + 'static { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream); + fn cancel_handler(self: &Arc, _id: RequestID) {} + + async fn recv_loop(self: Arc, mut read: R, debug_name: String) -> Result<(), Error> + where + R: AsyncReadExt + Unpin + Send + Sync, + { + let mut streams: HashMap = HashMap::new(); + loop { + trace!( + "recv_loop({}): in_progress = {:?}", + debug_name, + streams.iter().map(|(id, _)| id).collect::>() + ); + + let mut header_id = [0u8; RequestID::BITS as usize / 8]; + match read.read_exact(&mut header_id[..]).await { + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + }; + let id = RequestID::from_be_bytes(header_id); + + let mut header_size = [0u8; ChunkLength::BITS as usize / 8]; + read.read_exact(&mut header_size[..]).await?; + let size = ChunkLength::from_be_bytes(header_size); + + if size == CANCEL_REQUEST { + if let Some(mut stream) = streams.remove(&id) { + let _ = stream.send(Err(std::io::Error::new( + std::io::ErrorKind::Other, + "netapp: cancel requested", + ))); + stream.end(); + } + self.cancel_handler(id); + continue; + } + + let has_cont = (size & CHUNK_FLAG_HAS_CONTINUATION) != 0; + let is_error = (size & CHUNK_FLAG_ERROR) != 0; + let size = (size & CHUNK_LENGTH_MASK) as usize; + let mut next_slice = vec![0; size as usize]; + read.read_exact(&mut next_slice[..]).await?; + + let packet = if is_error { + let kind = u8_to_io_errorkind(next_slice[0]); + let msg = + std::str::from_utf8(&next_slice[1..]).unwrap_or(""); + debug!( + "recv_loop({}): got id {}, error {:?}: {}", + debug_name, id, kind, msg + ); + Some(Err(std::io::Error::new(kind, msg.to_string()))) + } else { + trace!( + "recv_loop({}): got id {}, size {}, has_cont {}", + debug_name, + id, + size, + has_cont + ); + if !next_slice.is_empty() { + Some(Ok(Bytes::from(next_slice))) + } else { + None + } + }; + + let mut sender = if let Some(send) = streams.remove(&(id)) { + send + } else { + let (send, recv) = mpsc::unbounded_channel(); + trace!("recv_loop({}): id {} is new channel", debug_name, id); + self.recv_handler( + id, + Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(recv)), + ); + Sender::new(send) + }; + + if let Some(packet) = packet { + // If we cannot put packet in channel, it means that the + // receiving end of the channel is disconnected. + // We still need to reach eos before dropping this sender + let _ = sender.send(packet); + } + + if has_cont { + assert!(!is_error); + streams.insert(id, sender); + } else { + trace!("recv_loop({}): close channel id {}", debug_name, id); + sender.end(); + } + } + Ok(()) + } +} diff --git a/src/net/send.rs b/src/net/send.rs new file mode 100644 index 00000000..6f1ac02c --- /dev/null +++ b/src/net/send.rs @@ -0,0 +1,359 @@ +use std::collections::{HashMap, VecDeque}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use bytes::{BufMut, Bytes, BytesMut}; +use log::*; + +use futures::{AsyncWriteExt, Future}; +use kuska_handshake::async_std::BoxStreamWrite; +use tokio::sync::mpsc; + +use crate::error::*; +use crate::message::*; +use crate::stream::*; + +// Messages are sent by chunks +// Chunk format: +// - u32 BE: request id (same for request and response) +// - u16 BE: chunk length + flags: +// CHUNK_FLAG_HAS_CONTINUATION when this is not the last chunk of the stream +// CHUNK_FLAG_ERROR if this chunk denotes an error +// (these two flags are exclusive, an error denotes the end of the stream) +// **special value** 0xFFFF indicates a CANCEL message +// - [u8; chunk_length], either +// - if not error: chunk data +// - if error: +// - u8: error kind, encoded using error::io_errorkind_to_u8 +// - rest: error message +// - absent for cancel message + +pub(crate) type RequestID = u32; +pub(crate) type ChunkLength = u16; + +pub(crate) const MAX_CHUNK_LENGTH: ChunkLength = 0x3FF0; +pub(crate) const CHUNK_FLAG_ERROR: ChunkLength = 0x4000; +pub(crate) const CHUNK_FLAG_HAS_CONTINUATION: ChunkLength = 0x8000; +pub(crate) const CHUNK_LENGTH_MASK: ChunkLength = 0x3FFF; +pub(crate) const CANCEL_REQUEST: ChunkLength = 0xFFFF; + +pub(crate) enum SendItem { + Stream(RequestID, RequestPriority, Option, ByteStream), + Cancel(RequestID), +} + +// ---- + +struct SendQueue { + items: Vec<(u8, SendQueuePriority)>, +} + +struct SendQueuePriority { + items: VecDeque, + order: HashMap>, +} + +struct SendQueueItem { + id: RequestID, + prio: RequestPriority, + order_tag: Option, + data: ByteStreamReader, + sent: usize, +} + +impl SendQueue { + fn new() -> Self { + Self { + items: Vec::with_capacity(64), + } + } + fn push(&mut self, item: SendQueueItem) { + let prio = item.prio; + let pos_prio = match self.items.binary_search_by(|(p, _)| p.cmp(&prio)) { + Ok(i) => i, + Err(i) => { + self.items.insert(i, (prio, SendQueuePriority::new())); + i + } + }; + self.items[pos_prio].1.push(item); + } + fn remove(&mut self, id: RequestID) { + for (_, prioq) in self.items.iter_mut() { + prioq.remove(id); + } + self.items.retain(|(_prio, q)| !q.is_empty()); + } + fn is_empty(&self) -> bool { + self.items.iter().all(|(_k, v)| v.is_empty()) + } + + // this is like an async fn, but hand implemented + fn next_ready(&mut self) -> SendQueuePollNextReady<'_> { + SendQueuePollNextReady { queue: self } + } +} + +impl SendQueuePriority { + fn new() -> Self { + Self { + items: VecDeque::new(), + order: HashMap::new(), + } + } + fn push(&mut self, item: SendQueueItem) { + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_vec = self.order.entry(stream).or_default(); + let i = order_vec.iter().take_while(|o2| **o2 < order).count(); + order_vec.insert(i, order); + } + self.items.push_back(item); + } + fn remove(&mut self, id: RequestID) { + if let Some(i) = self.items.iter().position(|x| x.id == id) { + let item = self.items.remove(i).unwrap(); + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_vec = self.order.get_mut(&stream).unwrap(); + let j = order_vec.iter().position(|x| *x == order).unwrap(); + order_vec.remove(j).unwrap(); + if order_vec.is_empty() { + self.order.remove(&stream); + } + } + } + } + fn is_empty(&self) -> bool { + self.items.is_empty() + } + fn poll_next_ready(&mut self, ctx: &mut Context<'_>) -> Poll<(RequestID, DataFrame)> { + // in step 1: poll only streams that have sent 0 bytes, we want to send them in priority + // as they most likely represent small requests to be sent first + // in step 2: poll all streams + for step in 0..2 { + for (j, item) in self.items.iter_mut().enumerate() { + if let Some(OrderTag(stream, order)) = item.order_tag { + if order > *self.order.get(&stream).unwrap().front().unwrap() { + continue; + } + } + + if step == 0 && item.sent > 0 { + continue; + } + + let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize); + if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) { + let id = item.id; + let eos = item.data.eos(); + + let packet = bytes_or_err.map_err(|e| match e { + ReadExactError::Stream(err) => err, + _ => unreachable!(), + }); + + let is_err = packet.is_err(); + let data_frame = DataFrame::from_packet(packet, !eos); + item.sent += data_frame.data().len(); + + if eos || is_err { + // If item had an order tag, remove it from the corresponding ordering list + if let Some(OrderTag(stream, order)) = item.order_tag { + let order_stream = self.order.get_mut(&stream).unwrap(); + assert_eq!(order_stream.pop_front(), Some(order)); + if order_stream.is_empty() { + self.order.remove(&stream); + } + } + // Remove item from sending queue + self.items.remove(j); + } else if step == 0 { + // Step 0 means that this stream had not sent any bytes yet. + // Now that it has, and it was not an EOS, we know that it is bigger + // than one chunk so move it at the end of the queue. + let item = self.items.remove(j).unwrap(); + self.items.push_back(item); + } + + return Poll::Ready((id, data_frame)); + } + } + } + + Poll::Pending + } + fn dump(&self, prio: u8) -> String { + self.items + .iter() + .map(|i| format!("[{} {} {:?} @{}]", prio, i.id, i.order_tag, i.sent)) + .collect::>() + .join(" ") + } +} + +struct SendQueuePollNextReady<'a> { + queue: &'a mut SendQueue, +} + +impl<'a> futures::Future for SendQueuePollNextReady<'a> { + type Output = (RequestID, DataFrame); + + fn poll(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + for (i, (_prio, items_at_prio)) in self.queue.items.iter_mut().enumerate() { + if let Poll::Ready(res) = items_at_prio.poll_next_ready(ctx) { + if items_at_prio.is_empty() { + self.queue.items.remove(i); + } + return Poll::Ready(res); + } + } + // If the queue is empty, this futures is eternally pending. + // This is ok because we use it in a select with another future + // that can interrupt it. + Poll::Pending + } +} + +enum DataFrame { + /// a fixed size buffer containing some data + a boolean indicating whether + /// there may be more data coming from this stream. Can be used for some + /// optimization. It's an error to set it to false if there is more data, but it is correct + /// (albeit sub-optimal) to set it to true if there is nothing coming after + Data(Bytes, bool), + /// An error code automatically signals the end of the stream + Error(Bytes), +} + +impl DataFrame { + fn from_packet(p: Packet, has_cont: bool) -> Self { + match p { + Ok(bytes) => { + assert!(bytes.len() <= MAX_CHUNK_LENGTH as usize); + Self::Data(bytes, has_cont) + } + Err(e) => { + let mut buf = BytesMut::new(); + buf.put_u8(io_errorkind_to_u8(e.kind())); + + let msg = format!("{}", e).into_bytes(); + if msg.len() > (MAX_CHUNK_LENGTH - 1) as usize { + buf.put(&msg[..(MAX_CHUNK_LENGTH - 1) as usize]); + } else { + buf.put(&msg[..]); + } + + Self::Error(buf.freeze()) + } + } + } + + fn header(&self) -> [u8; 2] { + let header_u16 = match self { + DataFrame::Data(data, false) => data.len() as u16, + DataFrame::Data(data, true) => data.len() as u16 | CHUNK_FLAG_HAS_CONTINUATION, + DataFrame::Error(msg) => msg.len() as u16 | CHUNK_FLAG_ERROR, + }; + ChunkLength::to_be_bytes(header_u16) + } + + fn data(&self) -> &[u8] { + match self { + DataFrame::Data(ref data, _) => &data[..], + DataFrame::Error(ref msg) => &msg[..], + } + } +} + +/// The SendLoop trait, which is implemented both by the client and the server +/// connection objects (ServerConna and ClientConn) adds a method `.send_loop()` +/// that takes a channel of messages to send and an asynchronous writer, +/// and sends messages from the channel to the async writer, putting them in a queue +/// before being sent and doing the round-robin sending strategy. +/// +/// The `.send_loop()` exits when the sending end of the channel is closed, +/// or if there is an error at any time writing to the async writer. +pub(crate) trait SendLoop: Sync { + async fn send_loop( + self: Arc, + msg_recv: mpsc::UnboundedReceiver, + mut write: BoxStreamWrite, + debug_name: String, + ) -> Result<(), Error> + where + W: AsyncWriteExt + Unpin + Send + Sync, + { + let mut sending = SendQueue::new(); + let mut msg_recv = Some(msg_recv); + while msg_recv.is_some() || !sending.is_empty() { + trace!( + "send_loop({}): queue = {:?}", + debug_name, + sending + .items + .iter() + .map(|(prio, i)| i.dump(*prio)) + .collect::>() + .join(" ; ") + ); + + let recv_fut = async { + if let Some(chan) = &mut msg_recv { + chan.recv().await + } else { + futures::future::pending().await + } + }; + let send_fut = sending.next_ready(); + + // recv_fut is cancellation-safe according to tokio doc, + // send_fut is cancellation-safe as implemented above? + tokio::select! { + biased; // always read incoming channel first if it has data + sth = recv_fut => { + match sth { + Some(SendItem::Stream(id, prio, order_tag, data)) => { + trace!("send_loop({}): add stream {} to send", debug_name, id); + sending.push(SendQueueItem { + id, + prio, + order_tag, + data: ByteStreamReader::new(data), + sent: 0, + }) + } + Some(SendItem::Cancel(id)) => { + trace!("send_loop({}): cancelling {}", debug_name, id); + sending.remove(id); + let header_id = RequestID::to_be_bytes(id); + write.write_all(&header_id[..]).await?; + write.write_all(&ChunkLength::to_be_bytes(CANCEL_REQUEST)).await?; + write.flush().await?; + } + None => { + msg_recv = None; + } + }; + } + (id, data) = send_fut => { + trace!( + "send_loop({}): id {}, send {} bytes, header_size {}", + debug_name, + id, + data.data().len(), + hex::encode(data.header()) + ); + + let header_id = RequestID::to_be_bytes(id); + write.write_all(&header_id[..]).await?; + + write.write_all(&data.header()).await?; + write.write_all(data.data()).await?; + write.flush().await?; + } + } + } + + let _ = write.goodbye().await; + Ok(()) + } +} diff --git a/src/net/server.rs b/src/net/server.rs new file mode 100644 index 00000000..fb6c6366 --- /dev/null +++ b/src/net/server.rs @@ -0,0 +1,220 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use arc_swap::ArcSwapOption; +use log::*; + +use futures::io::{AsyncReadExt, AsyncWriteExt}; +use kuska_handshake::async_std::{handshake_server, BoxStream}; +use tokio::net::TcpStream; +use tokio::select; +use tokio::sync::{mpsc, watch}; +use tokio_util::compat::*; + +#[cfg(feature = "telemetry")] +use opentelemetry::{ + trace::{FutureExt, Span, SpanKind, TraceContextExt, TraceId, Tracer}, + Context, KeyValue, +}; +#[cfg(feature = "telemetry")] +use opentelemetry_contrib::trace::propagator::binary::*; +#[cfg(feature = "telemetry")] +use rand::{thread_rng, Rng}; + +use crate::error::*; +use crate::message::*; +use crate::netapp::*; +use crate::recv::*; +use crate::send::*; +use crate::stream::*; +use crate::util::*; + +// The client and server connection structs (client.rs and server.rs) +// build upon the chunking mechanism which is exclusively contained +// in proto.rs. +// Here, we just care about sending big messages without size limit. +// The format of these messages is described below. +// Chunking happens independently. + +// Request message format (client -> server): +// - u8 priority +// - u8 path length +// - [u8; path length] path +// - [u8; *] data + +// Response message format (server -> client): +// - u8 response code +// - [u8; *] response + +pub(crate) struct ServerConn { + pub(crate) remote_addr: SocketAddr, + pub(crate) peer_id: NodeID, + + netapp: Arc, + + resp_send: ArcSwapOption>, + running_handlers: Mutex>>, +} + +impl ServerConn { + pub(crate) async fn run( + netapp: Arc, + socket: TcpStream, + must_exit: watch::Receiver, + ) -> Result<(), Error> { + let remote_addr = socket.peer_addr()?; + let mut socket = socket.compat(); + + // Do handshake to authenticate client + let handshake = handshake_server( + &mut socket, + netapp.netid.clone(), + netapp.id, + netapp.privkey.clone(), + ) + .await?; + let peer_id = handshake.peer_pk; + + debug!( + "Handshake complete (server) with {}@{}", + hex::encode(peer_id), + remote_addr + ); + + // Create BoxStream layer that encodes content + let (read, write) = socket.split(); + let (read, mut write) = + BoxStream::from_handshake(read, write, handshake, 0x8000).split_read_write(); + + // Before doing anything, send version tag, so that client + // can check and disconnect if version is wrong + write.write_all(&netapp.version_tag[..]).await?; + write.flush().await?; + + // Build and launch stuff that handles requests server-side + let (resp_send, resp_recv) = mpsc::unbounded_channel(); + + let conn = Arc::new(ServerConn { + netapp: netapp.clone(), + remote_addr, + peer_id, + resp_send: ArcSwapOption::new(Some(Arc::new(resp_send))), + running_handlers: Mutex::new(HashMap::new()), + }); + + netapp.connected_as_server(peer_id, conn.clone()); + + let debug_name = format!("SRV {}", hex::encode(&peer_id[..8])); + let debug_name_2 = debug_name.clone(); + + let conn2 = conn.clone(); + let recv_future = tokio::spawn(async move { + select! { + r = conn2.recv_loop(read, debug_name_2) => r, + _ = await_exit(must_exit) => Ok(()) + } + }); + let send_future = tokio::spawn(conn.clone().send_loop(resp_recv, write, debug_name)); + + recv_future.await.log_err("ServerConn recv_loop"); + conn.resp_send.store(None); + send_future.await.log_err("ServerConn send_loop"); + + netapp.disconnected_as_server(&peer_id, conn); + + Ok(()) + } + + async fn recv_handler_aux(self: &Arc, req_enc: ReqEnc) -> Result { + let path = String::from_utf8(req_enc.path.to_vec())?; + + let handler_opt = { + let endpoints = self.netapp.endpoints.read().unwrap(); + endpoints.get(&path).map(|e| e.clone_endpoint()) + }; + + if let Some(handler) = handler_opt { + cfg_if::cfg_if! { + if #[cfg(feature = "telemetry")] { + let tracer = opentelemetry::global::tracer("netapp"); + + let mut span = if !req_enc.telemetry_id.is_empty() { + let propagator = BinaryPropagator::new(); + let context = propagator.from_bytes(req_enc.telemetry_id.to_vec()); + let context = Context::new().with_remote_span_context(context); + tracer.span_builder(format!(">> RPC {}", path)) + .with_kind(SpanKind::Server) + .start_with_context(&tracer, &context) + } else { + let mut rng = thread_rng(); + let trace_id = TraceId::from_bytes(rng.gen()); + tracer + .span_builder(format!(">> RPC {}", path)) + .with_kind(SpanKind::Server) + .with_trace_id(trace_id) + .start(&tracer) + }; + span.set_attribute(KeyValue::new("path", path.to_string())); + span.set_attribute(KeyValue::new("len_query_msg", req_enc.msg.len() as i64)); + + handler.handle(req_enc, self.peer_id) + .with_context(Context::current_with_span(span)) + .await + } else { + handler.handle(req_enc, self.peer_id).await + } + } + } else { + Err(Error::NoHandler) + } + } +} + +impl SendLoop for ServerConn {} + +impl RecvLoop for ServerConn { + fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream) { + let resp_send = match self.resp_send.load_full() { + Some(c) => c, + None => return, + }; + + let mut rh = self.running_handlers.lock().unwrap(); + + let self2 = self.clone(); + let jh = tokio::spawn(async move { + debug!("server: recv_handler got {}", id); + + let (prio, resp_enc_result) = match ReqEnc::decode(stream).await { + Ok(req_enc) => (req_enc.prio, self2.recv_handler_aux(req_enc).await), + Err(e) => (PRIO_NORMAL, Err(e)), + }; + + debug!("server: sending response to {}", id); + + let (resp_stream, resp_order) = RespEnc::encode(resp_enc_result); + resp_send + .send(SendItem::Stream(id, prio, resp_order, resp_stream)) + .log_err("ServerConn recv_handler send resp bytes"); + + self2.running_handlers.lock().unwrap().remove(&id); + }); + + rh.insert(id, jh); + } + + fn cancel_handler(self: &Arc, id: RequestID) { + trace!("received cancel for request {}", id); + + // If the handler is still running, abort it now + if let Some(jh) = self.running_handlers.lock().unwrap().remove(&id) { + jh.abort(); + } + + // Inform the response sender that we don't need to send the response + if let Some(resp_send) = self.resp_send.load_full() { + let _ = resp_send.send(SendItem::Cancel(id)); + } + } +} diff --git a/src/net/stream.rs b/src/net/stream.rs new file mode 100644 index 00000000..c973f9a7 --- /dev/null +++ b/src/net/stream.rs @@ -0,0 +1,213 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use bytes::Bytes; + +use futures::Future; +use futures::{Stream, StreamExt}; +use tokio::io::AsyncRead; + +use crate::bytes_buf::BytesBuf; + +/// A stream of bytes (click to read more). +/// +/// When sent through Netapp, the Vec may be split in smaller chunk in such a way +/// consecutive Vec may get merged, but Vec and error code may not be reordered +/// +/// Items sent in the ByteStream may be errors of type `std::io::Error`. +/// An error indicates the end of the ByteStream: a reader should no longer read +/// after receiving an error, and a writer should stop writing after sending an error. +pub type ByteStream = Pin + Send + Sync>>; + +/// A packet sent in a ByteStream, which may contain either +/// a Bytes object or an error +pub type Packet = Result; + +// ---- + +/// A helper struct to read defined lengths of data from a BytesStream +pub struct ByteStreamReader { + stream: ByteStream, + buf: BytesBuf, + eos: bool, + err: Option, +} + +impl ByteStreamReader { + /// Creates a new `ByteStreamReader` from a `ByteStream` + pub fn new(stream: ByteStream) -> Self { + ByteStreamReader { + stream, + buf: BytesBuf::new(), + eos: false, + err: None, + } + } + + /// Read exactly `read_len` bytes from the underlying stream + /// (returns a future) + pub fn read_exact(&mut self, read_len: usize) -> ByteStreamReadExact<'_> { + ByteStreamReadExact { + reader: self, + read_len, + fail_on_eos: true, + } + } + + /// Read at most `read_len` bytes from the underlying stream, or less + /// if the end of the stream is reached (returns a future) + pub fn read_exact_or_eos(&mut self, read_len: usize) -> ByteStreamReadExact<'_> { + ByteStreamReadExact { + reader: self, + read_len, + fail_on_eos: false, + } + } + + /// Read exactly one byte from the underlying stream and returns it + /// as an u8 + pub async fn read_u8(&mut self) -> Result { + Ok(self.read_exact(1).await?[0]) + } + + /// Read exactly two bytes from the underlying stream and returns them as an u16 (using + /// big-endian decoding) + pub async fn read_u16(&mut self) -> Result { + let bytes = self.read_exact(2).await?; + let mut b = [0u8; 2]; + b.copy_from_slice(&bytes[..]); + Ok(u16::from_be_bytes(b)) + } + + /// Read exactly four bytes from the underlying stream and returns them as an u32 (using + /// big-endian decoding) + pub async fn read_u32(&mut self) -> Result { + let bytes = self.read_exact(4).await?; + let mut b = [0u8; 4]; + b.copy_from_slice(&bytes[..]); + Ok(u32::from_be_bytes(b)) + } + + /// Transforms the stream reader back into the underlying stream (starting + /// after everything that the reader has read) + pub fn into_stream(self) -> ByteStream { + let buf_stream = futures::stream::iter(self.buf.into_slices().into_iter().map(Ok)); + if let Some(err) = self.err { + Box::pin(buf_stream.chain(futures::stream::once(async move { Err(err) }))) + } else if self.eos { + Box::pin(buf_stream) + } else { + Box::pin(buf_stream.chain(self.stream)) + } + } + + /// Tries to fill the internal read buffer from the underlying stream if it is empty. + /// Calling this might be necessary to ensure that `.eos()` returns a correct + /// result, otherwise the reader might not be aware that the underlying + /// stream has nothing left to return. + pub async fn fill_buffer(&mut self) { + if self.buf.is_empty() { + let packet = self.stream.next().await; + self.add_stream_next(packet); + } + } + + /// Clears the internal read buffer and returns its content + pub fn take_buffer(&mut self) -> Bytes { + self.buf.take_all() + } + + /// Returns true if the end of the underlying stream has been reached + pub fn eos(&self) -> bool { + self.buf.is_empty() && self.eos + } + + fn try_get(&mut self, read_len: usize) -> Option { + self.buf.take_exact(read_len) + } + + fn add_stream_next(&mut self, packet: Option) { + match packet { + Some(Ok(slice)) => { + self.buf.extend(slice); + } + Some(Err(e)) => { + self.err = Some(e); + self.eos = true; + } + None => { + self.eos = true; + } + } + } +} + +/// The error kind that can be returned by `ByteStreamReader::read_exact` and +/// `ByteStreamReader::read_exact_or_eos` +pub enum ReadExactError { + /// The end of the stream was reached before the requested number of bytes could be read + UnexpectedEos, + /// The underlying data stream returned an IO error when trying to read + Stream(std::io::Error), +} + +/// The future returned by `ByteStreamReader::read_exact` and +/// `ByteStreamReader::read_exact_or_eos` +#[pin_project::pin_project] +pub struct ByteStreamReadExact<'a> { + #[pin] + reader: &'a mut ByteStreamReader, + read_len: usize, + fail_on_eos: bool, +} + +impl<'a> Future for ByteStreamReadExact<'a> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + loop { + if let Some(bytes) = this.reader.try_get(*this.read_len) { + return Poll::Ready(Ok(bytes)); + } + if let Some(err) = &this.reader.err { + let err = std::io::Error::new(err.kind(), format!("{}", err)); + return Poll::Ready(Err(ReadExactError::Stream(err))); + } + if this.reader.eos { + if *this.fail_on_eos { + return Poll::Ready(Err(ReadExactError::UnexpectedEos)); + } else { + return Poll::Ready(Ok(this.reader.take_buffer())); + } + } + + let next_packet = futures::ready!(this.reader.stream.as_mut().poll_next(cx)); + this.reader.add_stream_next(next_packet); + } + } +} + +// ---- + +/// Turns a `tokio::io::AsyncRead` asynchronous reader into a `ByteStream` +pub fn asyncread_stream(reader: R) -> ByteStream { + Box::pin(tokio_util::io::ReaderStream::new(reader)) +} + +/// Turns a `ByteStream` into a `tokio::io::AsyncRead` asynchronous reader +pub fn stream_asyncread(stream: ByteStream) -> impl AsyncRead + Send + Sync + 'static { + tokio_util::io::StreamReader::new(stream) +} + +/// Reads all of the content of a `ByteStream` into a BytesBuf +/// that contains everything +pub async fn read_stream_to_end(mut stream: ByteStream) -> Result { + let mut buf = BytesBuf::new(); + while let Some(part) = stream.next().await { + buf.extend(part?); + } + + Ok(buf) +} diff --git a/src/net/test.rs b/src/net/test.rs new file mode 100644 index 00000000..3cf446bd --- /dev/null +++ b/src/net/test.rs @@ -0,0 +1,118 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use tokio::select; +use tokio::sync::watch; + +use sodiumoxide::crypto::auth; +use sodiumoxide::crypto::sign::ed25519; + +use crate::netapp::*; +use crate::peering::*; +use crate::NodeID; + +#[tokio::test(flavor = "current_thread")] +async fn test_with_basic_scheduler() { + pretty_env_logger::init(); + run_test(19980).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_with_threaded_scheduler() { + run_test(19990).await +} + +async fn run_test(port_base: u16) { + select! { + _ = run_test_inner(port_base) => (), + _ = tokio::time::sleep(Duration::from_secs(20)) => panic!("timeout"), + } +} + +async fn run_test_inner(port_base: u16) { + let netid = auth::gen_key(); + + let (pk1, sk1) = ed25519::gen_keypair(); + let (pk2, sk2) = ed25519::gen_keypair(); + let (pk3, sk3) = ed25519::gen_keypair(); + + let addr1: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base); + let addr2: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base + 1); + let addr3: SocketAddr = SocketAddr::new("127.0.0.1".parse().unwrap(), port_base + 2); + + let (stop_tx, stop_rx) = watch::channel(false); + + let (thread1, _netapp1, peering1) = + run_netapp(netid.clone(), pk1, sk1, addr1, vec![], stop_rx.clone()); + tokio::time::sleep(Duration::from_secs(2)).await; + + // Connect second node and check it peers with everyone + let (thread2, _netapp2, peering2) = run_netapp( + netid.clone(), + pk2, + sk2, + addr2, + vec![(pk1, addr1)], + stop_rx.clone(), + ); + tokio::time::sleep(Duration::from_secs(3)).await; + + let pl1 = peering1.get_peer_list(); + println!("A pl1: {:?}", pl1); + assert_eq!(pl1.len(), 2); + + let pl2 = peering2.get_peer_list(); + println!("A pl2: {:?}", pl2); + assert_eq!(pl2.len(), 2); + + // Connect third node and check it peers with everyone + let (thread3, _netapp3, peering3) = + run_netapp(netid, pk3, sk3, addr3, vec![(pk2, addr2)], stop_rx.clone()); + tokio::time::sleep(Duration::from_secs(3)).await; + + let pl1 = peering1.get_peer_list(); + println!("B pl1: {:?}", pl1); + assert_eq!(pl1.len(), 3); + + let pl2 = peering2.get_peer_list(); + println!("B pl2: {:?}", pl2); + assert_eq!(pl2.len(), 3); + + let pl3 = peering3.get_peer_list(); + println!("B pl3: {:?}", pl3); + assert_eq!(pl3.len(), 3); + + // Send stop signal and wait for everyone to finish + stop_tx.send(true).unwrap(); + thread1.await.unwrap(); + thread2.await.unwrap(); + thread3.await.unwrap(); +} + +fn run_netapp( + netid: auth::Key, + _pk: NodeID, + sk: ed25519::SecretKey, + listen_addr: SocketAddr, + bootstrap_peers: Vec<(NodeID, SocketAddr)>, + must_exit: watch::Receiver, +) -> ( + tokio::task::JoinHandle<()>, + Arc, + Arc, +) { + let netapp = NetApp::new(0u64, netid, sk, None); + let peering = PeeringManager::new(netapp.clone(), bootstrap_peers, None); + + let peering2 = peering.clone(); + let netapp2 = netapp.clone(); + let fut = tokio::spawn(async move { + tokio::join!( + netapp2.listen(listen_addr, None, must_exit.clone()), + peering2.run(must_exit.clone()), + ); + }); + + (fut, netapp, peering) +} diff --git a/src/net/util.rs b/src/net/util.rs new file mode 100644 index 00000000..35a3be1e --- /dev/null +++ b/src/net/util.rs @@ -0,0 +1,96 @@ +use std::net::SocketAddr; + +use log::info; +use serde::Serialize; + +use tokio::sync::watch; + +use crate::netapp::*; + +/// Utility function: encodes any serializable value in MessagePack binary format +/// using the RMP library. +/// +/// Field names and variant names are included in the serialization. +/// This is used internally by the netapp communication protocol. +pub fn rmp_to_vec_all_named(val: &T) -> Result, rmp_serde::encode::Error> +where + T: Serialize + ?Sized, +{ + let mut wr = Vec::with_capacity(128); + let mut se = rmp_serde::Serializer::new(&mut wr).with_struct_map(); + val.serialize(&mut se)?; + Ok(wr) +} + +/// This async function returns only when a true signal was received +/// from a watcher that tells us when to exit. +/// +/// Useful in a select statement to interrupt another +/// future: +/// ```ignore +/// select!( +/// _ = a_long_task() => Success, +/// _ = await_exit(must_exit) => Interrupted, +/// ) +/// ``` +pub async fn await_exit(mut must_exit: watch::Receiver) { + while !*must_exit.borrow_and_update() { + if must_exit.changed().await.is_err() { + break; + } + } +} + +/// Creates a watch that contains `false`, and that changes +/// to `true` when a Ctrl+C signal is received. +pub fn watch_ctrl_c() -> watch::Receiver { + let (send_cancel, watch_cancel) = watch::channel(false); + tokio::spawn(async move { + tokio::signal::ctrl_c() + .await + .expect("failed to install CTRL+C signal handler"); + info!("Received CTRL+C, shutting down."); + send_cancel.send(true).unwrap(); + }); + watch_cancel +} + +/// Parse a peer's address including public key, written in the format: +/// `@:` +pub fn parse_peer_addr(peer: &str) -> Option<(NodeID, SocketAddr)> { + let delim = peer.find('@')?; + let (key, ip) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let ip = ip[1..].parse::().ok()?; + Some((pubkey, ip)) +} + +/// Parse and resolve a peer's address including public key, written in the format: +/// `@:` +pub fn parse_and_resolve_peer_addr(peer: &str) -> Option<(NodeID, Vec)> { + use std::net::ToSocketAddrs; + + let delim = peer.find('@')?; + let (key, host) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let hosts = host[1..].to_socket_addrs().ok()?.collect::>(); + if hosts.is_empty() { + return None; + } + Some((pubkey, hosts)) +} + +/// async version of parse_and_resolve_peer_addr +pub async fn parse_and_resolve_peer_addr_async(peer: &str) -> Option<(NodeID, Vec)> { + let delim = peer.find('@')?; + let (key, host) = peer.split_at(delim); + let pubkey = NodeID::from_slice(&hex::decode(key).ok()?)?; + let hosts = tokio::net::lookup_host(&host[1..]) + .await + .ok()? + .collect::>(); + if hosts.is_empty() { + return None; + } + Some((pubkey, hosts)) +} diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index dcf44f4a..e23f4bca 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,40 +14,40 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_util = { version = "0.8.2", path = "../util" } +format_table.workspace = true +garage_util.workspace = true +garage_net.workspace = true -arc-swap = "1.0" -bytes = "1.0" -gethostname = "0.2" -hex = "0.4" -tracing = "0.1" -rand = "0.8" -sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -systemstat = "0.2.3" +arc-swap.workspace = true +bytesize.workspace = true +gethostname.workspace = true +hex.workspace = true +ipnet.workspace = true +tracing.workspace = true +rand.workspace = true +itertools.workspace = true +sodiumoxide.workspace = true +nix.workspace = true -async-trait = "0.1.7" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -serde_json = "1.0" -err-derive = { version = "0.3", optional = true } +async-trait.workspace = true +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +thiserror = { workspace = true, optional = true } # newer version requires rust edition 2021 -kube = { version = "0.75", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"], optional = true } -k8s-openapi = { version = "0.16", features = ["v1_22"], optional = true } -schemars = { version = "0.8", optional = true } -reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls-manual-roots", "json"] } +kube = { workspace = true, optional = true } +k8s-openapi = { workspace = true, optional = true } +schemars = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } -pnet_datalink = "0.33" +pnet_datalink.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -tokio-stream = { version = "0.1", features = ["net"] } -opentelemetry = "0.17" - -netapp = { version = "0.5.2", features = ["telemetry"] } +futures.workspace = true +tokio.workspace = true +opentelemetry.workspace = true [features] kubernetes-discovery = [ "kube", "k8s-openapi", "schemars" ] -consul-discovery = [ "reqwest", "err-derive" ] +consul-discovery = [ "reqwest", "thiserror" ] system-libs = [ "sodiumoxide/use-pkg-config" ] diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index f85f789c..760e9fcb 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -3,21 +3,31 @@ use std::fs::File; use std::io::Read; use std::net::{IpAddr, SocketAddr}; -use err_derive::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; -use netapp::NodeID; +use garage_net::NodeID; +use garage_util::config::ConsulDiscoveryAPI; use garage_util::config::ConsulDiscoveryConfig; +const META_PREFIX: &str = "fr-deuxfleurs-garagehq"; + #[derive(Deserialize, Clone, Debug)] struct ConsulQueryEntry { #[serde(rename = "Address")] address: String, #[serde(rename = "ServicePort")] service_port: u16, - #[serde(rename = "NodeMeta")] - node_meta: HashMap, + #[serde(rename = "ServiceMeta")] + meta: HashMap, +} + +#[derive(Serialize, Clone, Debug)] +#[serde(untagged)] +enum PublishRequest { + Catalog(ConsulPublishEntry), + Service(ConsulPublishService), } #[derive(Serialize, Clone, Debug)] @@ -26,17 +36,31 @@ struct ConsulPublishEntry { node: String, #[serde(rename = "Address")] address: IpAddr, - #[serde(rename = "NodeMeta")] - node_meta: HashMap, #[serde(rename = "Service")] - service: ConsulPublishService, + service: ConsulPublishCatalogService, +} + +#[derive(Serialize, Clone, Debug)] +struct ConsulPublishCatalogService { + #[serde(rename = "ID")] + service_id: String, + #[serde(rename = "Service")] + service_name: String, + #[serde(rename = "Tags")] + tags: Vec, + #[serde(rename = "Meta")] + meta: HashMap, + #[serde(rename = "Address")] + address: IpAddr, + #[serde(rename = "Port")] + port: u16, } #[derive(Serialize, Clone, Debug)] struct ConsulPublishService { #[serde(rename = "ID")] service_id: String, - #[serde(rename = "Service")] + #[serde(rename = "Name")] service_name: String, #[serde(rename = "Tags")] tags: Vec, @@ -44,10 +68,11 @@ struct ConsulPublishService { address: IpAddr, #[serde(rename = "Port")] port: u16, + #[serde(rename = "Meta")] + meta: HashMap, } // ---- - pub struct ConsulDiscovery { config: ConsulDiscoveryConfig, client: reqwest::Client, @@ -55,44 +80,48 @@ pub struct ConsulDiscovery { impl ConsulDiscovery { pub fn new(config: ConsulDiscoveryConfig) -> Result { - let client = match (&config.client_cert, &config.client_key) { - (Some(client_cert), Some(client_key)) => { - let mut client_cert_buf = vec![]; - File::open(client_cert)?.read_to_end(&mut client_cert_buf)?; + let mut builder: reqwest::ClientBuilder = reqwest::Client::builder().use_rustls_tls(); + if config.tls_skip_verify { + builder = builder.danger_accept_invalid_certs(true); + } else if let Some(ca_cert) = &config.ca_cert { + let mut ca_cert_buf = vec![]; + File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?; + builder = + builder.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?); + } - let mut client_key_buf = vec![]; - File::open(client_key)?.read_to_end(&mut client_key_buf)?; + match &config.api { + ConsulDiscoveryAPI::Catalog => match (&config.client_cert, &config.client_key) { + (Some(client_cert), Some(client_key)) => { + let mut client_cert_buf = vec![]; + File::open(client_cert)?.read_to_end(&mut client_cert_buf)?; - let identity = reqwest::Identity::from_pem( - &[&client_cert_buf[..], &client_key_buf[..]].concat()[..], - )?; + let mut client_key_buf = vec![]; + File::open(client_key)?.read_to_end(&mut client_key_buf)?; - if config.tls_skip_verify { - reqwest::Client::builder() - .use_rustls_tls() - .danger_accept_invalid_certs(true) - .identity(identity) - .build()? - } else if let Some(ca_cert) = &config.ca_cert { - let mut ca_cert_buf = vec![]; - File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?; + let identity = reqwest::Identity::from_pem( + &[&client_cert_buf[..], &client_key_buf[..]].concat()[..], + )?; - reqwest::Client::builder() - .use_rustls_tls() - .add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?) - .identity(identity) - .build()? - } else { - reqwest::Client::builder() - .use_rustls_tls() - .identity(identity) - .build()? + builder = builder.identity(identity); + } + (None, None) => {} + _ => return Err(ConsulError::InvalidTLSConfig), + }, + ConsulDiscoveryAPI::Agent => { + if let Some(token) = &config.token { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + "x-consul-token", + reqwest::header::HeaderValue::from_str(&token)?, + ); + builder = builder.default_headers(headers); } } - (None, None) => reqwest::Client::new(), - _ => return Err(ConsulError::InvalidTLSConfig), }; + let client: reqwest::Client = builder.build()?; + Ok(Self { client, config }) } @@ -111,15 +140,15 @@ impl ConsulDiscovery { for ent in entries { let ip = ent.address.parse::().ok(); let pubkey = ent - .node_meta - .get("pubkey") + .meta + .get(&format!("{}-pubkey", META_PREFIX)) .and_then(|k| hex::decode(k).ok()) .and_then(|k| NodeID::from_slice(&k[..])); if let (Some(ip), Some(pubkey)) = (ip, pubkey) { ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); } else { warn!( - "Could not process node spec from Consul: {:?} (invalid IP or public key)", + "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", ent ); } @@ -138,29 +167,49 @@ impl ConsulDiscovery { rpc_public_addr: SocketAddr, ) -> Result<(), ConsulError> { let node = format!("garage:{}", hex::encode(&node_id[..8])); + let tags = [ + vec!["advertised-by-garage".into(), hostname.into()], + self.config.tags.clone(), + ] + .concat(); - let advertisement = ConsulPublishEntry { - node: node.clone(), - address: rpc_public_addr.ip(), - node_meta: [ - ("pubkey".to_string(), hex::encode(node_id)), - ("hostname".to_string(), hostname.to_string()), - ] - .iter() - .cloned() - .collect(), - service: ConsulPublishService { + let mut meta = self.config.meta.clone().unwrap_or_default(); + meta.insert(format!("{}-pubkey", META_PREFIX), hex::encode(node_id)); + meta.insert(format!("{}-hostname", META_PREFIX), hostname.to_string()); + + let url = format!( + "{}/v1/{}", + self.config.consul_http_addr, + (match &self.config.api { + ConsulDiscoveryAPI::Catalog => "catalog/register", + ConsulDiscoveryAPI::Agent => "agent/service/register?replace-existing-checks", + }) + ); + + let req = self.client.put(&url); + let advertisement: PublishRequest = match &self.config.api { + ConsulDiscoveryAPI::Catalog => PublishRequest::Catalog(ConsulPublishEntry { + node: node.clone(), + address: rpc_public_addr.ip(), + service: ConsulPublishCatalogService { + service_id: node.clone(), + service_name: self.config.service_name.clone(), + tags, + meta: meta.clone(), + address: rpc_public_addr.ip(), + port: rpc_public_addr.port(), + }, + }), + ConsulDiscoveryAPI::Agent => PublishRequest::Service(ConsulPublishService { service_id: node.clone(), service_name: self.config.service_name.clone(), - tags: vec!["advertised-by-garage".into(), hostname.into()], + tags, + meta, address: rpc_public_addr.ip(), port: rpc_public_addr.port(), - }, + }), }; - - let url = format!("{}/v1/catalog/register", self.config.consul_http_addr); - - let http = self.client.put(&url).json(&advertisement).send().await?; + let http = req.json(&advertisement).send().await?; http.error_for_status()?; Ok(()) @@ -170,10 +219,12 @@ impl ConsulDiscovery { /// Regroup all Consul discovery errors #[derive(Debug, Error)] pub enum ConsulError { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] std::io::Error), - #[error(display = "HTTP error: {}", _0)] - Reqwest(#[error(source)] reqwest::Error), - #[error(display = "Invalid Consul TLS configuration")] + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("HTTP error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("Invalid Consul TLS configuration")] InvalidTLSConfig, + #[error("Token error: {0}")] + Token(#[from] reqwest::header::InvalidHeaderValue), } diff --git a/src/rpc/kubernetes.rs b/src/rpc/kubernetes.rs index 63c6567d..85254bb5 100644 --- a/src/rpc/kubernetes.rs +++ b/src/rpc/kubernetes.rs @@ -10,7 +10,7 @@ use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomRe use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use netapp::NodeID; +use garage_net::NodeID; use garage_util::config::KubernetesDiscoveryConfig; diff --git a/src/rpc/layout.rs b/src/rpc/layout.rs deleted file mode 100644 index 1030e3a6..00000000 --- a/src/rpc/layout.rs +++ /dev/null @@ -1,669 +0,0 @@ -use std::cmp::Ordering; -use std::collections::{HashMap, HashSet}; - -use serde::{Deserialize, Serialize}; - -use garage_util::crdt::{AutoCrdt, Crdt, LwwMap}; -use garage_util::data::*; -use garage_util::encode::nonversioned_encode; -use garage_util::error::*; - -use crate::ring::*; - -/// The layout of the cluster, i.e. the list of roles -/// which are assigned to each cluster node -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ClusterLayout { - pub version: u64, - - pub replication_factor: usize, - pub roles: LwwMap, - - /// node_id_vec: a vector of node IDs with a role assigned - /// in the system (this includes gateway nodes). - /// The order here is different than the vec stored by `roles`, because: - /// 1. non-gateway nodes are first so that they have lower numbers - /// 2. nodes that don't have a role are excluded (but they need to - /// stay in the CRDT as tombstones) - pub node_id_vec: Vec, - /// the assignation of data partitions to node, the values - /// are indices in node_id_vec - #[serde(with = "serde_bytes")] - pub ring_assignation_data: Vec, - - /// Role changes which are staged for the next version of the layout - pub staging: LwwMap, - pub staging_hash: Hash, -} - -impl garage_util::migrate::InitialFormat for ClusterLayout {} - -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] -pub struct NodeRoleV(pub Option); - -impl AutoCrdt for NodeRoleV { - const WARN_IF_DIFFERENT: bool = true; -} - -/// The user-assigned roles of cluster nodes -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] -pub struct NodeRole { - /// Datacenter at which this entry belong. This information might be used to perform a better - /// geodistribution - pub zone: String, - /// The (relative) capacity of the node - /// If this is set to None, the node does not participate in storing data for the system - /// and is only active as an API gateway to other nodes - pub capacity: Option, - /// A set of tags to recognize the node - pub tags: Vec, -} - -impl NodeRole { - pub fn capacity_string(&self) -> String { - match self.capacity { - Some(c) => format!("{}", c), - None => "gateway".to_string(), - } - } -} - -impl ClusterLayout { - pub fn new(replication_factor: usize) -> Self { - let empty_lwwmap = LwwMap::new(); - let empty_lwwmap_hash = blake2sum(&nonversioned_encode(&empty_lwwmap).unwrap()[..]); - - ClusterLayout { - version: 0, - replication_factor, - roles: LwwMap::new(), - node_id_vec: Vec::new(), - ring_assignation_data: Vec::new(), - staging: empty_lwwmap, - staging_hash: empty_lwwmap_hash, - } - } - - pub fn merge(&mut self, other: &ClusterLayout) -> bool { - match other.version.cmp(&self.version) { - Ordering::Greater => { - *self = other.clone(); - true - } - Ordering::Equal => { - self.staging.merge(&other.staging); - - let new_staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]); - let changed = new_staging_hash != self.staging_hash; - - self.staging_hash = new_staging_hash; - - changed - } - Ordering::Less => false, - } - } - - pub fn apply_staged_changes(mut self, version: Option) -> Result { - match version { - None => { - let error = r#" -Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. -To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. - "#; - return Err(Error::Message(error.into())); - } - Some(v) => { - if v != self.version + 1 { - return Err(Error::Message("Invalid new layout version".into())); - } - } - } - - self.roles.merge(&self.staging); - self.roles.retain(|(_, _, v)| v.0.is_some()); - - if !self.calculate_partition_assignation() { - return Err(Error::Message("Could not calculate new assignation of partitions to nodes. This can happen if there are less nodes than the desired number of copies of your data (see the replication_mode configuration parameter).".into())); - } - - self.staging.clear(); - self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]); - - self.version += 1; - - Ok(self) - } - - pub fn revert_staged_changes(mut self, version: Option) -> Result { - match version { - None => { - let error = r#" -Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. -To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. - "#; - return Err(Error::Message(error.into())); - } - Some(v) => { - if v != self.version + 1 { - return Err(Error::Message("Invalid new layout version".into())); - } - } - } - - self.staging.clear(); - self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]); - - self.version += 1; - - Ok(self) - } - - /// Returns a list of IDs of nodes that currently have - /// a role in the cluster - pub fn node_ids(&self) -> &[Uuid] { - &self.node_id_vec[..] - } - - pub fn num_nodes(&self) -> usize { - self.node_id_vec.len() - } - - /// Returns the role of a node in the layout - pub fn node_role(&self, node: &Uuid) -> Option<&NodeRole> { - match self.roles.get(node) { - Some(NodeRoleV(Some(v))) => Some(v), - _ => None, - } - } - - /// Check a cluster layout for internal consistency - /// returns true if consistent, false if error - pub fn check(&self) -> bool { - // Check that the hash of the staging data is correct - let staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]); - if staging_hash != self.staging_hash { - return false; - } - - // Check that node_id_vec contains the correct list of nodes - let mut expected_nodes = self - .roles - .items() - .iter() - .filter(|(_, _, v)| v.0.is_some()) - .map(|(id, _, _)| *id) - .collect::>(); - expected_nodes.sort(); - let mut node_id_vec = self.node_id_vec.clone(); - node_id_vec.sort(); - if expected_nodes != node_id_vec { - return false; - } - - // Check that the assignation data has the correct length - if self.ring_assignation_data.len() != (1 << PARTITION_BITS) * self.replication_factor { - return false; - } - - // Check that the assigned nodes are correct identifiers - // of nodes that are assigned a role - // and that role is not the role of a gateway nodes - for x in self.ring_assignation_data.iter() { - if *x as usize >= self.node_id_vec.len() { - return false; - } - let node = self.node_id_vec[*x as usize]; - match self.roles.get(&node) { - Some(NodeRoleV(Some(x))) if x.capacity.is_some() => (), - _ => return false, - } - } - - true - } - - /// Calculate an assignation of partitions to nodes - pub fn calculate_partition_assignation(&mut self) -> bool { - let (configured_nodes, zones) = self.configured_nodes_and_zones(); - let n_zones = zones.len(); - - println!("Calculating updated partition assignation, this may take some time..."); - println!(); - - // Get old partition assignation - let old_partitions = self.parse_assignation_data(); - - // Start new partition assignation with nodes from old assignation where it is relevant - let mut partitions = old_partitions - .iter() - .map(|old_part| { - let mut new_part = PartitionAss::new(); - for node in old_part.nodes.iter() { - if let Some(role) = node.1 { - if role.capacity.is_some() { - new_part.add(None, n_zones, node.0, role); - } - } - } - new_part - }) - .collect::>(); - - // In various cases, not enough nodes will have been added for all partitions - // in the step above (e.g. due to node removals, or new zones being added). - // Here we add more nodes to make a complete (but sub-optimal) assignation, - // using an initial partition assignation that is calculated using the multi-dc maglev trick - match self.initial_partition_assignation() { - Some(initial_partitions) => { - for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) { - for _ in 0..2 { - for (id, info) in ipart.nodes.iter() { - if part.nodes.len() < self.replication_factor { - part.add(None, n_zones, id, info.unwrap()); - } - } - } - assert!(part.nodes.len() == self.replication_factor); - } - } - None => { - // Not enough nodes in cluster to build a correct assignation. - // Signal it by returning an error. - return false; - } - } - - // Calculate how many partitions each node should ideally store, - // and how many partitions they are storing with the current assignation - // This defines our target for which we will optimize in the following loop. - let total_capacity = configured_nodes - .iter() - .map(|(_, info)| info.capacity.unwrap_or(0)) - .sum::() as usize; - let total_partitions = self.replication_factor * (1 << PARTITION_BITS); - let target_partitions_per_node = configured_nodes - .iter() - .map(|(id, info)| { - ( - *id, - info.capacity.unwrap_or(0) as usize * total_partitions / total_capacity, - ) - }) - .collect::>(); - - let mut partitions_per_node = self.partitions_per_node(&partitions[..]); - - println!("Target number of partitions per node:"); - for (node, npart) in target_partitions_per_node.iter() { - println!("{:?}\t{}", node, npart); - } - println!(); - - // Shuffle partitions between nodes so that nodes will reach (or better approach) - // their target number of stored partitions - loop { - let mut option = None; - for (i, part) in partitions.iter_mut().enumerate() { - for (irm, (idrm, _)) in part.nodes.iter().enumerate() { - let errratio = |node, parts| { - let tgt = *target_partitions_per_node.get(node).unwrap() as f32; - (parts - tgt) / tgt - }; - let square = |x| x * x; - - let partsrm = partitions_per_node.get(*idrm).cloned().unwrap_or(0) as f32; - - for (idadd, infoadd) in configured_nodes.iter() { - // skip replacing a node by itself - // and skip replacing by gateway nodes - if idadd == idrm || infoadd.capacity.is_none() { - continue; - } - - // We want to try replacing node idrm by node idadd - // if that brings us close to our goal. - let partsadd = partitions_per_node.get(*idadd).cloned().unwrap_or(0) as f32; - let oldcost = square(errratio(*idrm, partsrm) - errratio(*idadd, partsadd)); - let newcost = - square(errratio(*idrm, partsrm - 1.) - errratio(*idadd, partsadd + 1.)); - if newcost >= oldcost { - // not closer to our goal - continue; - } - let gain = oldcost - newcost; - - let mut newpart = part.clone(); - - newpart.nodes.remove(irm); - if !newpart.add(None, n_zones, idadd, infoadd) { - continue; - } - assert!(newpart.nodes.len() == self.replication_factor); - - if !old_partitions[i] - .is_valid_transition_to(&newpart, self.replication_factor) - { - continue; - } - - if option - .as_ref() - .map(|(old_gain, _, _, _, _)| gain > *old_gain) - .unwrap_or(true) - { - option = Some((gain, i, idadd, idrm, newpart)); - } - } - } - } - if let Some((_gain, i, idadd, idrm, newpart)) = option { - *partitions_per_node.entry(idadd).or_insert(0) += 1; - *partitions_per_node.get_mut(idrm).unwrap() -= 1; - partitions[i] = newpart; - } else { - break; - } - } - - // Check we completed the assignation correctly - // (this is a set of checks for the algorithm's consistency) - assert!(partitions.len() == (1 << PARTITION_BITS)); - assert!(partitions - .iter() - .all(|p| p.nodes.len() == self.replication_factor)); - - let new_partitions_per_node = self.partitions_per_node(&partitions[..]); - assert!(new_partitions_per_node == partitions_per_node); - - // Show statistics - println!("New number of partitions per node:"); - for (node, npart) in partitions_per_node.iter() { - let tgt = *target_partitions_per_node.get(node).unwrap(); - let pct = 100f32 * (*npart as f32) / (tgt as f32); - println!("{:?}\t{}\t({}% of {})", node, npart, pct as i32, tgt); - } - println!(); - - let mut diffcount = HashMap::new(); - for (oldpart, newpart) in old_partitions.iter().zip(partitions.iter()) { - let nminus = oldpart.txtplus(newpart); - let nplus = newpart.txtplus(oldpart); - if nminus != "[...]" || nplus != "[...]" { - let tup = (nminus, nplus); - *diffcount.entry(tup).or_insert(0) += 1; - } - } - if diffcount.is_empty() { - println!("No data will be moved between nodes."); - } else { - let mut diffcount = diffcount.into_iter().collect::>(); - diffcount.sort(); - println!("Number of partitions that move:"); - for ((nminus, nplus), npart) in diffcount { - println!("\t{}\t{} -> {}", npart, nminus, nplus); - } - } - println!(); - - // Calculate and save new assignation data - let (nodes, assignation_data) = - self.compute_assignation_data(&configured_nodes[..], &partitions[..]); - - self.node_id_vec = nodes; - self.ring_assignation_data = assignation_data; - - true - } - - fn initial_partition_assignation(&self) -> Option>> { - let (configured_nodes, zones) = self.configured_nodes_and_zones(); - let n_zones = zones.len(); - - // Create a vector of partition indices (0 to 2**PARTITION_BITS-1) - let partitions_idx = (0usize..(1usize << PARTITION_BITS)).collect::>(); - - // Prepare ring - let mut partitions: Vec = partitions_idx - .iter() - .map(|_i| PartitionAss::new()) - .collect::>(); - - // Create MagLev priority queues for each node - let mut queues = configured_nodes - .iter() - .filter(|(_id, info)| info.capacity.is_some()) - .map(|(node_id, node_info)| { - let mut parts = partitions_idx - .iter() - .map(|i| { - let part_data = - [&u16::to_be_bytes(*i as u16)[..], node_id.as_slice()].concat(); - (*i, fasthash(&part_data[..])) - }) - .collect::>(); - parts.sort_by_key(|(_i, h)| *h); - let parts_i = parts.iter().map(|(i, _h)| *i).collect::>(); - (node_id, node_info, parts_i, 0) - }) - .collect::>(); - - let max_capacity = configured_nodes - .iter() - .filter_map(|(_, node_info)| node_info.capacity) - .fold(0, std::cmp::max); - - // Fill up ring - for rep in 0..self.replication_factor { - queues.sort_by_key(|(ni, _np, _q, _p)| { - let queue_data = [&u16::to_be_bytes(rep as u16)[..], ni.as_slice()].concat(); - fasthash(&queue_data[..]) - }); - - for (_, _, _, pos) in queues.iter_mut() { - *pos = 0; - } - - let mut remaining = partitions_idx.len(); - while remaining > 0 { - let remaining0 = remaining; - for i_round in 0..max_capacity { - for (node_id, node_info, q, pos) in queues.iter_mut() { - if i_round >= node_info.capacity.unwrap() { - continue; - } - for (pos2, &qv) in q.iter().enumerate().skip(*pos) { - if partitions[qv].add(Some(rep + 1), n_zones, node_id, node_info) { - remaining -= 1; - *pos = pos2 + 1; - break; - } - } - } - } - if remaining == remaining0 { - // No progress made, exit - return None; - } - } - } - - Some(partitions) - } - - fn configured_nodes_and_zones(&self) -> (Vec<(&Uuid, &NodeRole)>, HashSet<&str>) { - let configured_nodes = self - .roles - .items() - .iter() - .filter(|(_id, _, info)| info.0.is_some()) - .map(|(id, _, info)| (id, info.0.as_ref().unwrap())) - .collect::>(); - - let zones = configured_nodes - .iter() - .filter(|(_id, info)| info.capacity.is_some()) - .map(|(_id, info)| info.zone.as_str()) - .collect::>(); - - (configured_nodes, zones) - } - - fn compute_assignation_data<'a>( - &self, - configured_nodes: &[(&'a Uuid, &'a NodeRole)], - partitions: &[PartitionAss<'a>], - ) -> (Vec, Vec) { - assert!(partitions.len() == (1 << PARTITION_BITS)); - - // Make a canonical order for nodes - let mut nodes = configured_nodes - .iter() - .filter(|(_id, info)| info.capacity.is_some()) - .map(|(id, _)| **id) - .collect::>(); - let nodes_rev = nodes - .iter() - .enumerate() - .map(|(i, id)| (*id, i as CompactNodeType)) - .collect::>(); - - let mut assignation_data = vec![]; - for partition in partitions.iter() { - assert!(partition.nodes.len() == self.replication_factor); - for (id, _) in partition.nodes.iter() { - assignation_data.push(*nodes_rev.get(id).unwrap()); - } - } - - nodes.extend( - configured_nodes - .iter() - .filter(|(_id, info)| info.capacity.is_none()) - .map(|(id, _)| **id), - ); - - (nodes, assignation_data) - } - - fn parse_assignation_data(&self) -> Vec> { - if self.ring_assignation_data.len() == self.replication_factor * (1 << PARTITION_BITS) { - // If the previous assignation data is correct, use that - let mut partitions = vec![]; - for i in 0..(1 << PARTITION_BITS) { - let mut part = PartitionAss::new(); - for node_i in self.ring_assignation_data - [i * self.replication_factor..(i + 1) * self.replication_factor] - .iter() - { - let node_id = &self.node_id_vec[*node_i as usize]; - - if let Some(NodeRoleV(Some(info))) = self.roles.get(node_id) { - part.nodes.push((node_id, Some(info))); - } else { - part.nodes.push((node_id, None)); - } - } - partitions.push(part); - } - partitions - } else { - // Otherwise start fresh - (0..(1 << PARTITION_BITS)) - .map(|_| PartitionAss::new()) - .collect() - } - } - - fn partitions_per_node<'a>(&self, partitions: &[PartitionAss<'a>]) -> HashMap<&'a Uuid, usize> { - let mut partitions_per_node = HashMap::<&Uuid, usize>::new(); - for p in partitions.iter() { - for (id, _) in p.nodes.iter() { - *partitions_per_node.entry(*id).or_insert(0) += 1; - } - } - partitions_per_node - } -} - -// ---- Internal structs for partition assignation in layout ---- - -#[derive(Clone)] -struct PartitionAss<'a> { - nodes: Vec<(&'a Uuid, Option<&'a NodeRole>)>, -} - -impl<'a> PartitionAss<'a> { - fn new() -> Self { - Self { nodes: Vec::new() } - } - - fn nplus(&self, other: &PartitionAss<'a>) -> usize { - self.nodes - .iter() - .filter(|x| !other.nodes.contains(x)) - .count() - } - - fn txtplus(&self, other: &PartitionAss<'a>) -> String { - let mut nodes = self - .nodes - .iter() - .filter(|x| !other.nodes.contains(x)) - .map(|x| format!("{:?}", x.0)) - .collect::>(); - nodes.sort(); - if self.nodes.iter().any(|x| other.nodes.contains(x)) { - nodes.push("...".into()); - } - format!("[{}]", nodes.join(" ")) - } - - fn is_valid_transition_to(&self, other: &PartitionAss<'a>, replication_factor: usize) -> bool { - let min_keep_nodes_per_part = (replication_factor + 1) / 2; - let n_removed = self.nplus(other); - - if self.nodes.len() <= min_keep_nodes_per_part { - n_removed == 0 - } else { - n_removed <= self.nodes.len() - min_keep_nodes_per_part - } - } - - // add is a key function in creating a PartitionAss, i.e. the list of nodes - // to which a partition is assigned. It tries to add a certain node id to the - // assignation, but checks that doing so is compatible with the NECESSARY - // condition that the partition assignation must be dispersed over different - // zones (datacenters) if enough zones exist. This is why it takes a n_zones - // parameter, which is the total number of zones that have existing nodes: - // if nodes in the assignation already cover all n_zones zones, then any node - // that is not yet in the assignation can be added. Otherwise, only nodes - // that are in a new zone can be added. - fn add( - &mut self, - target_len: Option, - n_zones: usize, - node: &'a Uuid, - role: &'a NodeRole, - ) -> bool { - if let Some(tl) = target_len { - if self.nodes.len() != tl - 1 { - return false; - } - } - - let p_zns = self - .nodes - .iter() - .map(|(_id, info)| info.unwrap().zone.as_str()) - .collect::>(); - if (p_zns.len() < n_zones && !p_zns.contains(&role.zone.as_str())) - || (p_zns.len() == n_zones && !self.nodes.iter().any(|(id, _)| *id == node)) - { - self.nodes.push((node, Some(role))); - true - } else { - false - } - } -} diff --git a/src/rpc/layout/graph_algo.rs b/src/rpc/layout/graph_algo.rs new file mode 100644 index 00000000..29d4a043 --- /dev/null +++ b/src/rpc/layout/graph_algo.rs @@ -0,0 +1,405 @@ +//! This module deals with graph algorithms. +//! It is used in layout.rs to build the partition to node assignment. + +use rand::prelude::{SeedableRng, SliceRandom}; +use std::cmp::{max, min}; +use std::collections::HashMap; +use std::collections::VecDeque; + +/// Vertex data structures used in all the graphs used in layout.rs. +/// usize parameters correspond to node/zone/partitions ids. +/// To understand the vertex roles below, please refer to the formal description +/// of the layout computation algorithm. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Vertex { + Source, + Pup(usize), // The vertex p+ of partition p + Pdown(usize), // The vertex p- of partition p + PZ(usize, usize), // The vertex corresponding to x_(partition p, zone z) + N(usize), // The vertex corresponding to node n + Sink, +} + +/// Edge data structure for the flow algorithm. +#[derive(Clone, Copy, Debug)] +pub struct FlowEdge { + cap: u64, // flow maximal capacity of the edge + flow: i64, // flow value on the edge + dest: usize, // destination vertex id + rev: usize, // index of the reversed edge (v, self) in the edge list of vertex v +} + +/// Edge data structure for the detection of negative cycles. +#[derive(Clone, Copy, Debug)] +pub struct WeightedEdge { + w: i64, // weight of the edge + dest: usize, +} + +pub trait Edge: Clone + Copy {} +impl Edge for FlowEdge {} +impl Edge for WeightedEdge {} + +/// Struct for the graph structure. We do encapsulation here to be able to both +/// provide user friendly Vertex enum to address vertices, and to use internally usize +/// indices and Vec instead of HashMap in the graph algorithm to optimize execution speed. +pub struct Graph { + vertex_to_id: HashMap, + id_to_vertex: Vec, + + // The graph is stored as an adjacency list + graph: Vec>, +} + +pub type CostFunction = HashMap<(Vertex, Vertex), i64>; + +impl Graph { + pub fn new(vertices: &[Vertex]) -> Self { + let mut map = HashMap::::new(); + for (i, vert) in vertices.iter().enumerate() { + map.insert(*vert, i); + } + Graph:: { + vertex_to_id: map, + id_to_vertex: vertices.to_vec(), + graph: vec![Vec::::new(); vertices.len()], + } + } + + fn get_vertex_id(&self, v: &Vertex) -> Result { + self.vertex_to_id + .get(v) + .cloned() + .ok_or_else(|| format!("The graph does not contain vertex {:?}", v)) + } +} + +impl Graph { + /// This function adds a directed edge to the graph with capacity c, and the + /// corresponding reversed edge with capacity 0. + pub fn add_edge(&mut self, u: Vertex, v: Vertex, c: u64) -> Result<(), String> { + let idu = self.get_vertex_id(&u)?; + let idv = self.get_vertex_id(&v)?; + if idu == idv { + return Err("Cannot add edge from vertex to itself in flow graph".into()); + } + + let rev_u = self.graph[idu].len(); + let rev_v = self.graph[idv].len(); + self.graph[idu].push(FlowEdge { + cap: c, + dest: idv, + flow: 0, + rev: rev_v, + }); + self.graph[idv].push(FlowEdge { + cap: 0, + dest: idu, + flow: 0, + rev: rev_u, + }); + Ok(()) + } + + /// This function returns the list of vertices that receive a positive flow from + /// vertex v. + pub fn get_positive_flow_from(&self, v: Vertex) -> Result, String> { + let idv = self.get_vertex_id(&v)?; + let mut result = Vec::::new(); + for edge in self.graph[idv].iter() { + if edge.flow > 0 { + result.push(self.id_to_vertex[edge.dest]); + } + } + Ok(result) + } + + /// This function returns the value of the flow outgoing from v. + pub fn get_outflow(&self, v: Vertex) -> Result { + let idv = self.get_vertex_id(&v)?; + let mut result = 0; + for edge in self.graph[idv].iter() { + result += max(0, edge.flow); + } + Ok(result) + } + + /// This function computes the flow total value by computing the outgoing flow + /// from the source. + pub fn get_flow_value(&mut self) -> Result { + self.get_outflow(Vertex::Source) + } + + /// This function shuffles the order of the edge lists. It keeps the ids of the + /// reversed edges consistent. + fn shuffle_edges(&mut self) { + // We use deterministic randomness so that the layout calculation algorithm + // will output the same thing every time it is run. This way, the results + // pre-calculated in `garage layout show` will match exactly those used + // in practice with `garage layout apply` + let mut rng = rand::rngs::StdRng::from_seed([0x12u8; 32]); + for i in 0..self.graph.len() { + self.graph[i].shuffle(&mut rng); + // We need to update the ids of the reverse edges. + for j in 0..self.graph[i].len() { + let target_v = self.graph[i][j].dest; + let target_rev = self.graph[i][j].rev; + self.graph[target_v][target_rev].rev = j; + } + } + } + + /// Computes an upper bound of the flow on the graph + pub fn flow_upper_bound(&self) -> Result { + let idsource = self.get_vertex_id(&Vertex::Source)?; + let mut flow_upper_bound = 0; + for edge in self.graph[idsource].iter() { + flow_upper_bound += edge.cap; + } + Ok(flow_upper_bound) + } + + /// This function computes the maximal flow using Dinic's algorithm. It starts with + /// the flow values already present in the graph. So it is possible to add some edge to + /// the graph, compute a flow, add other edges, update the flow. + pub fn compute_maximal_flow(&mut self) -> Result<(), String> { + let idsource = self.get_vertex_id(&Vertex::Source)?; + let idsink = self.get_vertex_id(&Vertex::Sink)?; + + let nb_vertices = self.graph.len(); + + let flow_upper_bound = self.flow_upper_bound()?; + + // To ensure the dispersion of the associations generated by the + // assignment, we shuffle the neighbours of the nodes. Hence, + // the vertices do not consider their neighbours in the same order. + self.shuffle_edges(); + + // We run Dinic's max flow algorithm + loop { + // We build the level array from Dinic's algorithm. + let mut level = vec![None; nb_vertices]; + + let mut fifo = VecDeque::new(); + fifo.push_back((idsource, 0)); + while let Some((id, lvl)) = fifo.pop_front() { + if level[id].is_none() { + // it means id has not yet been reached + level[id] = Some(lvl); + for edge in self.graph[id].iter() { + if edge.cap as i64 - edge.flow > 0 { + fifo.push_back((edge.dest, lvl + 1)); + } + } + } + } + if level[idsink].is_none() { + // There is no residual flow + break; + } + // Now we run DFS respecting the level array + let mut next_nbd = vec![0; nb_vertices]; + let mut lifo = Vec::new(); + + lifo.push((idsource, flow_upper_bound)); + + while let Some((id, f)) = lifo.last().cloned() { + if id == idsink { + // The DFS reached the sink, we can add a + // residual flow. + lifo.pop(); + while let Some((id, _)) = lifo.pop() { + let nbd = next_nbd[id]; + self.graph[id][nbd].flow += f as i64; + let id_rev = self.graph[id][nbd].dest; + let nbd_rev = self.graph[id][nbd].rev; + self.graph[id_rev][nbd_rev].flow -= f as i64; + } + lifo.push((idsource, flow_upper_bound)); + continue; + } + // else we did not reach the sink + let nbd = next_nbd[id]; + if nbd >= self.graph[id].len() { + // There is nothing to explore from id anymore + lifo.pop(); + if let Some((parent, _)) = lifo.last() { + next_nbd[*parent] += 1; + } + continue; + } + // else we can try to send flow from id to its nbd + let new_flow = min( + f as i64, + self.graph[id][nbd].cap as i64 - self.graph[id][nbd].flow, + ) as u64; + if new_flow == 0 { + next_nbd[id] += 1; + continue; + } + if let (Some(lvldest), Some(lvlid)) = (level[self.graph[id][nbd].dest], level[id]) { + if lvldest <= lvlid { + // We cannot send flow to nbd. + next_nbd[id] += 1; + continue; + } + } + // otherwise, we send flow to nbd. + lifo.push((self.graph[id][nbd].dest, new_flow)); + } + } + Ok(()) + } + + /// This function takes a flow, and a cost function on the edges, and tries to find an + /// equivalent flow with a better cost, by finding improving overflow cycles. It uses + /// as subroutine the Bellman Ford algorithm run up to path_length. + /// We assume that the cost of edge (u,v) is the opposite of the cost of (v,u), and + /// only one needs to be present in the cost function. + pub fn optimize_flow_with_cost( + &mut self, + cost: &CostFunction, + path_length: usize, + ) -> Result<(), String> { + // We build the weighted graph g where we will look for negative cycle + let mut gf = self.build_cost_graph(cost)?; + let mut cycles = gf.list_negative_cycles(path_length); + while !cycles.is_empty() { + // we enumerate negative cycles + for c in cycles.iter() { + for i in 0..c.len() { + // We add one flow unit to the edge (u,v) of cycle c + let idu = self.vertex_to_id[&c[i]]; + let idv = self.vertex_to_id[&c[(i + 1) % c.len()]]; + for j in 0..self.graph[idu].len() { + // since idu appears at most once in the cycles, we enumerate every + // edge at most once. + let edge = self.graph[idu][j]; + if edge.dest == idv { + self.graph[idu][j].flow += 1; + self.graph[idv][edge.rev].flow -= 1; + break; + } + } + } + } + + gf = self.build_cost_graph(cost)?; + cycles = gf.list_negative_cycles(path_length); + } + Ok(()) + } + + /// Construct the weighted graph G_f from the flow and the cost function + fn build_cost_graph(&self, cost: &CostFunction) -> Result, String> { + let mut g = Graph::::new(&self.id_to_vertex); + let nb_vertices = self.id_to_vertex.len(); + for i in 0..nb_vertices { + for edge in self.graph[i].iter() { + if edge.cap as i64 - edge.flow > 0 { + // It is possible to send overflow through this edge + let u = self.id_to_vertex[i]; + let v = self.id_to_vertex[edge.dest]; + if cost.contains_key(&(u, v)) { + g.add_edge(u, v, cost[&(u, v)])?; + } else if cost.contains_key(&(v, u)) { + g.add_edge(u, v, -cost[&(v, u)])?; + } else { + g.add_edge(u, v, 0)?; + } + } + } + } + Ok(g) + } +} + +impl Graph { + /// This function adds a single directed weighted edge to the graph. + pub fn add_edge(&mut self, u: Vertex, v: Vertex, w: i64) -> Result<(), String> { + let idu = self.get_vertex_id(&u)?; + let idv = self.get_vertex_id(&v)?; + self.graph[idu].push(WeightedEdge { w, dest: idv }); + Ok(()) + } + + /// This function lists the negative cycles it manages to find after path_length + /// iterations of the main loop of the Bellman-Ford algorithm. For the classical + /// algorithm, path_length needs to be equal to the number of vertices. However, + /// for particular graph structures like in our case, the algorithm is still correct + /// when path_length is the length of the longest possible simple path. + /// See the formal description of the algorithm for more details. + fn list_negative_cycles(&self, path_length: usize) -> Vec> { + let nb_vertices = self.graph.len(); + + // We start with every vertex at distance 0 of some imaginary extra -1 vertex. + let mut distance = vec![0; nb_vertices]; + // The prev vector collects for every vertex from where does the shortest path come + let mut prev = vec![None; nb_vertices]; + + for _ in 0..path_length + 1 { + for id in 0..nb_vertices { + for e in self.graph[id].iter() { + if distance[id] + e.w < distance[e.dest] { + distance[e.dest] = distance[id] + e.w; + prev[e.dest] = Some(id); + } + } + } + } + + // If self.graph contains a negative cycle, then at this point the graph described + // by prev (which is a directed 1-forest/functional graph) + // must contain a cycle. We list the cycles of prev. + let cycles_prev = cycles_of_1_forest(&prev); + + // Remark that the cycle in prev is in the reverse order compared to the cycle + // in the graph. Thus the .rev(). + return cycles_prev + .iter() + .map(|cycle| { + cycle + .iter() + .rev() + .map(|id| self.id_to_vertex[*id]) + .collect() + }) + .collect(); + } +} + +/// This function returns the list of cycles of a directed 1 forest. It does not +/// check for the consistency of the input. +fn cycles_of_1_forest(forest: &[Option]) -> Vec> { + let mut cycles = Vec::>::new(); + let mut time_of_discovery = vec![None; forest.len()]; + + for t in 0..forest.len() { + let mut id = t; + // while we are on a valid undiscovered node + while time_of_discovery[id].is_none() { + time_of_discovery[id] = Some(t); + if let Some(i) = forest[id] { + id = i; + } else { + break; + } + } + if forest[id].is_some() && time_of_discovery[id] == Some(t) { + // We discovered an id that we explored at this iteration t. + // It means we are on a cycle + let mut cy = vec![id; 1]; + let mut id2 = id; + while let Some(id_next) = forest[id2] { + id2 = id_next; + if id2 != id { + cy.push(id2); + } else { + break; + } + } + cycles.push(cy); + } + } + cycles +} diff --git a/src/rpc/layout/helper.rs b/src/rpc/layout/helper.rs new file mode 100644 index 00000000..c08a5629 --- /dev/null +++ b/src/rpc/layout/helper.rs @@ -0,0 +1,304 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use serde::{Deserialize, Serialize}; + +use garage_util::data::*; + +use super::*; +use crate::replication_mode::*; + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct RpcLayoutDigest { + /// Cluster layout version + pub current_version: u64, + /// Number of active layout versions + pub active_versions: usize, + /// Hash of cluster layout update trackers + pub trackers_hash: Hash, + /// Hash of cluster layout staging data + pub staging_hash: Hash, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct SyncLayoutDigest { + current: u64, + ack_map_min: u64, + min_stored: u64, +} + +pub struct LayoutHelper { + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + layout: Option, + + // cached values + ack_map_min: u64, + sync_map_min: u64, + + all_nodes: Vec, + all_nongateway_nodes: Vec, + + trackers_hash: Hash, + staging_hash: Hash, + is_check_ok: bool, + + // ack lock: counts in-progress write operations for each + // layout version ; we don't increase the ack update tracker + // while this lock is nonzero + pub(crate) ack_lock: HashMap, +} + +impl LayoutHelper { + pub fn new( + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + mut layout: LayoutHistory, + mut ack_lock: HashMap, + ) -> Self { + // In the new() function of the helper, we do a bunch of cleanup + // and calculations on the layout history to make sure things are + // correct and we have rapid access to important values such as + // the layout versions to use when reading to ensure consistency. + + if consistency_mode != ConsistencyMode::Consistent { + // Fast path for when no consistency is required. + // In this case we only need to keep the last version of the layout, + // we don't care about coordinating stuff in the cluster. + layout.keep_current_version_only(); + } + + layout.cleanup_old_versions(); + + let all_nodes = layout.get_all_nodes(); + let all_nongateway_nodes = layout.get_all_nongateway_nodes(); + + layout.clamp_update_trackers(&all_nodes); + + let min_version = layout.min_stored(); + + // ack_map_min is the minimum value of ack_map among all nodes + // in the cluster (gateway, non-gateway, current and previous layouts). + // It is the highest layout version which all of these nodes have + // acknowledged, indicating that they are aware of it and are no + // longer processing write operations that did not take it into account. + let ack_map_min = layout + .update_trackers + .ack_map + .min_among(&all_nodes, min_version); + + // sync_map_min is the minimum value of sync_map among storage nodes + // in the cluster (non-gateway nodes only, current and previous layouts). + // It is the highest layout version for which we know that all relevant + // storage nodes have fulfilled a sync, and therefore it is safe to + // use a read quorum within that layout to ensure consistency. + // Gateway nodes are excluded here because they hold no relevant data + // (they store the bucket and access key tables, but we don't have + // consistency on those). + // This value is calculated using quorums to allow progress even + // if not all nodes have successfully completed a sync. + let sync_map_min = + layout.calculate_sync_map_min_with_quorum(replication_factor, &all_nongateway_nodes); + + let trackers_hash = layout.calculate_trackers_hash(); + let staging_hash = layout.calculate_staging_hash(); + + ack_lock.retain(|_, cnt| *cnt.get_mut() > 0); + ack_lock + .entry(layout.current().version) + .or_insert(AtomicUsize::new(0)); + + let is_check_ok = layout.check().is_ok(); + + LayoutHelper { + replication_factor, + consistency_mode, + layout: Some(layout), + ack_map_min, + sync_map_min, + all_nodes, + all_nongateway_nodes, + trackers_hash, + staging_hash, + ack_lock, + is_check_ok, + } + } + + // ------------------ single updating function -------------- + + pub(crate) fn update(&mut self, f: F) -> bool + where + F: FnOnce(&mut LayoutHistory) -> bool, + { + let changed = f(self.layout.as_mut().unwrap()); + if changed { + *self = Self::new( + self.replication_factor, + self.consistency_mode, + self.layout.take().unwrap(), + std::mem::take(&mut self.ack_lock), + ); + } + changed + } + + // ------------------ read helpers --------------- + + pub fn inner(&self) -> &LayoutHistory { + self.layout.as_ref().unwrap() + } + + pub fn current(&self) -> &LayoutVersion { + self.inner().current() + } + + pub fn versions(&self) -> &[LayoutVersion] { + &self.inner().versions + } + + pub fn is_check_ok(&self) -> bool { + self.is_check_ok + } + + /// Return all nodes that have a role (gateway or storage) + /// in one of the currently active layout versions + pub fn all_nodes(&self) -> &[Uuid] { + &self.all_nodes + } + + /// Return all nodes that are configured to store data + /// in one of the currently active layout versions + pub fn all_nongateway_nodes(&self) -> &[Uuid] { + &self.all_nongateway_nodes + } + + pub fn ack_map_min(&self) -> u64 { + self.ack_map_min + } + + pub fn sync_map_min(&self) -> u64 { + self.sync_map_min + } + + pub fn sync_digest(&self) -> SyncLayoutDigest { + SyncLayoutDigest { + current: self.current().version, + ack_map_min: self.ack_map_min(), + min_stored: self.inner().min_stored(), + } + } + + pub fn read_nodes_of(&self, position: &Hash) -> Vec { + let sync_min = self.sync_map_min; + let version = self + .versions() + .iter() + .find(|x| x.version == sync_min) + .or(self.versions().last()) + .unwrap(); + version + .nodes_of(position, version.replication_factor) + .collect() + } + + pub fn storage_sets_of(&self, position: &Hash) -> Vec> { + self.versions() + .iter() + .map(|x| x.nodes_of(position, x.replication_factor).collect()) + .collect() + } + + pub fn storage_nodes_of(&self, position: &Hash) -> Vec { + let mut ret = vec![]; + for version in self.versions().iter() { + ret.extend(version.nodes_of(position, version.replication_factor)); + } + ret.sort(); + ret.dedup(); + ret + } + + pub fn current_storage_nodes_of(&self, position: &Hash) -> Vec { + let ver = self.current(); + ver.nodes_of(position, ver.replication_factor).collect() + } + + pub fn trackers_hash(&self) -> Hash { + self.trackers_hash + } + + pub fn staging_hash(&self) -> Hash { + self.staging_hash + } + + pub fn digest(&self) -> RpcLayoutDigest { + RpcLayoutDigest { + current_version: self.current().version, + active_versions: self.versions().len(), + trackers_hash: self.trackers_hash, + staging_hash: self.staging_hash, + } + } + + // ------------------ helpers for update tracking --------------- + + pub(crate) fn update_update_trackers(&mut self, local_node_id: Uuid) { + // Ensure trackers for this node's values are up-to-date + + // 1. Acknowledge the last layout version which is not currently + // locked by an in-progress write operation + self.update_ack_to_max_free(local_node_id); + + // 2. Assume the data on this node is sync'ed up at least to + // the first layout version in the history + let first_version = self.inner().min_stored(); + self.update(|layout| { + layout + .update_trackers + .sync_map + .set_max(local_node_id, first_version) + }); + + // 3. Acknowledge everyone has synced up to min(self.sync_map) + let sync_map_min = self.sync_map_min; + self.update(|layout| { + layout + .update_trackers + .sync_ack_map + .set_max(local_node_id, sync_map_min) + }); + + debug!("ack_map: {:?}", self.inner().update_trackers.ack_map); + debug!("sync_map: {:?}", self.inner().update_trackers.sync_map); + debug!( + "sync_ack_map: {:?}", + self.inner().update_trackers.sync_ack_map + ); + } + + pub(crate) fn update_ack_to_max_free(&mut self, local_node_id: Uuid) -> bool { + let max_free = self + .versions() + .iter() + .map(|x| x.version) + .skip_while(|v| { + self.ack_lock + .get(v) + .map(|x| x.load(Ordering::Relaxed) == 0) + .unwrap_or(true) + }) + .next() + .unwrap_or(self.current().version); + let changed = self.update(|layout| { + layout + .update_trackers + .ack_map + .set_max(local_node_id, max_free) + }); + if changed { + info!("ack_until updated to {}", max_free); + } + changed + } +} diff --git a/src/rpc/layout/history.rs b/src/rpc/layout/history.rs new file mode 100644 index 00000000..574c50c2 --- /dev/null +++ b/src/rpc/layout/history.rs @@ -0,0 +1,317 @@ +use std::collections::HashSet; + +use garage_util::crdt::{Crdt, Lww, LwwMap}; +use garage_util::data::*; +use garage_util::encode::nonversioned_encode; +use garage_util::error::*; + +use super::*; +use crate::replication_mode::*; + +impl LayoutHistory { + pub fn new(replication_factor: ReplicationFactor) -> Self { + let version = LayoutVersion::new(replication_factor.into()); + + let staging = LayoutStaging { + parameters: Lww::::new(version.parameters), + roles: LwwMap::new(), + }; + + LayoutHistory { + versions: vec![version], + old_versions: vec![], + update_trackers: Default::default(), + staging: Lww::raw(0, staging), + } + } + + // ------------------ who stores what now? --------------- + + /// Returns the layout version with the highest number + pub fn current(&self) -> &LayoutVersion { + self.versions.last().as_ref().unwrap() + } + + /// Returns the version number of the oldest layout version still active + pub fn min_stored(&self) -> u64 { + self.versions.first().as_ref().unwrap().version + } + + /// Calculate the set of all nodes that have a role (gateway or storage) + /// in one of the currently active layout versions + pub fn get_all_nodes(&self) -> Vec { + if self.versions.len() == 1 { + self.versions[0].all_nodes().to_vec() + } else { + let set = self + .versions + .iter() + .flat_map(|x| x.all_nodes()) + .collect::>(); + set.into_iter().copied().collect::>() + } + } + + /// Calculate the set of all nodes that are configured to store data + /// in one of the currently active layout versions + pub(crate) fn get_all_nongateway_nodes(&self) -> Vec { + if self.versions.len() == 1 { + self.versions[0].nongateway_nodes().to_vec() + } else { + let set = self + .versions + .iter() + .flat_map(|x| x.nongateway_nodes()) + .collect::>(); + set.into_iter().copied().collect::>() + } + } + + // ---- housekeeping (all invoked by LayoutHelper) ---- + + pub(crate) fn keep_current_version_only(&mut self) { + while self.versions.len() > 1 { + let removed = self.versions.remove(0); + self.old_versions.push(removed); + } + } + + pub(crate) fn cleanup_old_versions(&mut self) { + // If there are invalid versions before valid versions, remove them + if self.versions.len() > 1 && self.current().check().is_ok() { + while self.versions.len() > 1 && self.versions.first().unwrap().check().is_err() { + let removed = self.versions.remove(0); + info!( + "Layout history: pruning old invalid version {}", + removed.version + ); + } + } + + // If there are old versions that no one is reading from anymore, + // remove them (keep them in self.old_versions). + // ASSUMPTION: we only care about where nodes in the current layout version + // are reading from, as we assume older nodes are being discarded. + let current_nodes = &self.current().node_id_vec; + let min_version = self.min_stored(); + let sync_ack_map_min = self + .update_trackers + .sync_ack_map + .min_among(current_nodes, min_version); + while self.min_stored() < sync_ack_map_min { + assert!(self.versions.len() > 1); + let removed = self.versions.remove(0); + info!( + "Layout history: moving version {} to old_versions", + removed.version + ); + self.old_versions.push(removed); + } + + while self.old_versions.len() > OLD_VERSION_COUNT { + let removed = self.old_versions.remove(0); + info!("Layout history: removing old_version {}", removed.version); + } + } + + pub(crate) fn clamp_update_trackers(&mut self, nodes: &[Uuid]) { + let min_v = self.min_stored(); + for node in nodes { + self.update_trackers.ack_map.set_max(*node, min_v); + self.update_trackers.sync_map.set_max(*node, min_v); + self.update_trackers.sync_ack_map.set_max(*node, min_v); + } + } + + pub(crate) fn calculate_sync_map_min_with_quorum( + &self, + replication_factor: ReplicationFactor, + all_nongateway_nodes: &[Uuid], + ) -> u64 { + // This function calculates the minimum layout version from which + // it is safe to read if we want to maintain read-after-write consistency. + // In the general case the computation can be a bit expensive so + // we try to optimize it in several ways. + + // If there is only one layout version, we know that's the one + // we need to read from. + if self.versions.len() == 1 { + return self.current().version; + } + + let quorum = replication_factor.write_quorum(ConsistencyMode::Consistent); + + let min_version = self.min_stored(); + let global_min = self + .update_trackers + .sync_map + .min_among(all_nongateway_nodes, min_version); + + // If the write quorums are equal to the total number of nodes, + // i.e. no writes can succeed while they are not written to all nodes, + // then we must in all case wait for all nodes to complete a sync. + // This is represented by reading from the layout with version + // number global_min, the smallest layout version for which all nodes + // have completed a sync. + if quorum == self.current().replication_factor { + return global_min; + } + + // In the general case, we need to look at all write sets for all partitions, + // and find a safe layout version to read for that partition. We then + // take the minimum value among all partition as the safe layout version + // to read in all cases (the layout version to which all reads are directed). + let mut current_min = self.current().version; + let mut sets_done = HashSet::>::new(); + + for (_, p_hash) in self.current().partitions() { + for v in self.versions.iter() { + if v.version == self.current().version { + // We don't care about whether nodes in the latest layout version + // have completed a sync or not, as the sync is push-only + // and by definition nodes in the latest layout version do not + // hold data that must be pushed to nodes in the latest layout + // version, since that's the same version (any data that's + // already in the latest version is assumed to have been written + // by an operation that ensured a quorum of writes within + // that version). + continue; + } + + // Determine set of nodes for partition p in layout version v. + // Sort the node set to avoid duplicate computations. + let mut set = v + .nodes_of(&p_hash, v.replication_factor) + .collect::>(); + set.sort(); + + // If this set was already processed, skip it. + if sets_done.contains(&set) { + continue; + } + + // Find the value of the sync update trackers that is the + // highest possible minimum within a quorum of nodes. + let mut sync_values = set + .iter() + .map(|x| self.update_trackers.sync_map.get(x, min_version)) + .collect::>(); + sync_values.sort(); + let set_min = sync_values[sync_values.len() - quorum]; + if set_min < current_min { + current_min = set_min; + } + // defavorable case, we know we are at the smallest possible version, + // so we can stop early + assert!(current_min >= global_min); + if current_min == global_min { + return current_min; + } + + // Add set to already processed sets + sets_done.insert(set); + } + } + + current_min + } + + pub(crate) fn calculate_trackers_hash(&self) -> Hash { + blake2sum(&nonversioned_encode(&self.update_trackers).unwrap()[..]) + } + + pub(crate) fn calculate_staging_hash(&self) -> Hash { + blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]) + } + + // ================== updates to layout, public interface =================== + + pub fn merge(&mut self, other: &LayoutHistory) -> bool { + // If our current layout version is completely out-of-date, + // forget everything we know and replace it by incoming layout data. + if self.current().version < other.min_stored() { + *self = other.clone(); + return true; + } + + let mut changed = false; + + // Add any new versions to history + for v2 in other.versions.iter() { + if v2.version == self.current().version + 1 { + // This is the next version, add it to our version list + self.versions.push(v2.clone()); + changed = true; + } else if let Some(v1) = self.versions.iter().find(|v| v.version == v2.version) { + // Version is already present, check consistency + if v1 != v2 { + error!("Inconsistent layout histories: different layout compositions for version {}. Your cluster will be broken as long as this layout version is not replaced.", v2.version); + } + } else { + // This is an older version + assert!(v2.version < self.min_stored()); + } + } + + // Merge trackers + let c = self.update_trackers.merge(&other.update_trackers); + changed = changed || c; + + // Merge staged layout changes + if self.staging != other.staging { + let prev_staging = self.staging.clone(); + self.staging.merge(&other.staging); + changed = changed || self.staging != prev_staging; + } + + changed + } + + pub fn apply_staged_changes(mut self, version: Option) -> Result<(Self, Message), Error> { + match version { + None => { + let error = r#" +Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. +To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. + "#; + return Err(Error::Message(error.into())); + } + Some(v) => { + if v != self.current().version + 1 { + return Err(Error::Message("Invalid new layout version".into())); + } + } + } + + // Compute new version and add it to history + let (new_version, msg) = self + .current() + .clone() + .calculate_next_version(self.staging.get())?; + + self.versions.push(new_version); + self.cleanup_old_versions(); + + // Reset the staged layout changes + self.staging.update(LayoutStaging { + parameters: self.staging.get().parameters.clone(), + roles: LwwMap::new(), + }); + + Ok((self, msg)) + } + + pub fn revert_staged_changes(mut self) -> Result { + self.staging.update(LayoutStaging { + parameters: Lww::new(self.current().parameters), + roles: LwwMap::new(), + }); + + Ok(self) + } + + pub fn check(&self) -> Result<(), String> { + // TODO: anything more ? + self.current().check() + } +} diff --git a/src/rpc/layout/manager.rs b/src/rpc/layout/manager.rs new file mode 100644 index 00000000..bb8000bd --- /dev/null +++ b/src/rpc/layout/manager.rs @@ -0,0 +1,379 @@ +use std::collections::HashMap; +use std::sync::{atomic::Ordering, Arc, Mutex, RwLock, RwLockReadGuard}; +use std::time::Duration; + +use tokio::sync::Notify; + +use garage_net::endpoint::Endpoint; +use garage_net::peering::PeeringManager; +use garage_net::NodeID; + +use garage_util::config::Config; +use garage_util::data::*; +use garage_util::error::*; +use garage_util::persister::Persister; + +use super::*; +use crate::replication_mode::*; +use crate::rpc_helper::*; +use crate::system::*; + +pub struct LayoutManager { + node_id: Uuid, + replication_factor: ReplicationFactor, + persist_cluster_layout: Persister, + + layout: Arc>, + pub(crate) change_notify: Arc, + + table_sync_version: Mutex>, + + pub(crate) rpc_helper: RpcHelper, + system_endpoint: Arc>, +} + +impl LayoutManager { + pub fn new( + config: &Config, + node_id: NodeID, + system_endpoint: Arc>, + peering: Arc, + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, + ) -> Result, Error> { + let persist_cluster_layout: Persister = + Persister::new(&config.metadata_dir, "cluster_layout"); + + let cluster_layout = match persist_cluster_layout.load() { + Ok(x) => { + if x.current().replication_factor != replication_factor.replication_factor() { + return Err(Error::Message(format!( + "Previous cluster layout has replication factor {}, which is different than the one specified in the config file ({}). The previous cluster layout can be purged, if you know what you are doing, simply by deleting the `cluster_layout` file in your metadata directory.", + x.current().replication_factor, + replication_factor.replication_factor() + ))); + } + x + } + Err(e) => { + info!( + "No valid previous cluster layout stored ({}), starting fresh.", + e + ); + LayoutHistory::new(replication_factor) + } + }; + + let mut cluster_layout = LayoutHelper::new( + replication_factor, + consistency_mode, + cluster_layout, + Default::default(), + ); + cluster_layout.update_update_trackers(node_id.into()); + + let layout = Arc::new(RwLock::new(cluster_layout)); + let change_notify = Arc::new(Notify::new()); + + let rpc_helper = RpcHelper::new( + node_id.into(), + peering, + layout.clone(), + config.rpc_timeout_msec.map(Duration::from_millis), + ); + + Ok(Arc::new(Self { + node_id: node_id.into(), + replication_factor, + persist_cluster_layout, + layout, + change_notify, + table_sync_version: Mutex::new(HashMap::new()), + system_endpoint, + rpc_helper, + })) + } + + // ---- PUBLIC INTERFACE ---- + + pub fn layout(&self) -> RwLockReadGuard<'_, LayoutHelper> { + self.layout.read().unwrap() + } + + pub async fn update_cluster_layout( + self: &Arc, + layout: &LayoutHistory, + ) -> Result<(), Error> { + self.handle_advertise_cluster_layout(layout).await?; + Ok(()) + } + + pub fn add_table(&self, table_name: &'static str) { + let first_version = self.layout().versions().first().unwrap().version; + + self.table_sync_version + .lock() + .unwrap() + .insert(table_name.to_string(), first_version); + } + + pub fn sync_table_until(self: &Arc, table_name: &'static str, version: u64) { + let mut table_sync_version = self.table_sync_version.lock().unwrap(); + *table_sync_version.get_mut(table_name).unwrap() = version; + let sync_until = table_sync_version.iter().map(|(_, v)| *v).min().unwrap(); + drop(table_sync_version); + + let mut layout = self.layout.write().unwrap(); + if layout.update(|l| l.update_trackers.sync_map.set_max(self.node_id, sync_until)) { + info!("sync_until updated to {}", sync_until); + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers( + layout.inner().update_trackers.clone(), + )); + } + } + + fn ack_new_version(self: &Arc) { + let mut layout = self.layout.write().unwrap(); + if layout.update_ack_to_max_free(self.node_id) { + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers( + layout.inner().update_trackers.clone(), + )); + } + } + + // ---- ACK LOCKING ---- + + pub fn write_sets_of(self: &Arc, position: &Hash) -> WriteLock>> { + let layout = self.layout(); + let version = layout.current().version; + let nodes = layout.storage_sets_of(position); + layout + .ack_lock + .get(&version) + .unwrap() + .fetch_add(1, Ordering::Relaxed); + WriteLock::new(version, self, nodes) + } + + // ---- INTERNALS --- + + fn merge_layout(&self, adv: &LayoutHistory) -> Option { + let mut layout = self.layout.write().unwrap(); + let prev_digest = layout.digest(); + let prev_layout_check = layout.is_check_ok(); + + if !prev_layout_check || adv.check().is_ok() { + if layout.update(|l| l.merge(adv)) { + layout.update_update_trackers(self.node_id); + if prev_layout_check && !layout.is_check_ok() { + panic!("Merged two correct layouts and got an incorrect layout."); + } + assert!(layout.digest() != prev_digest); + return Some(layout.inner().clone()); + } + } + + None + } + + fn merge_layout_trackers(&self, adv: &UpdateTrackers) -> Option { + let mut layout = self.layout.write().unwrap(); + let prev_digest = layout.digest(); + + if layout.inner().update_trackers != *adv { + if layout.update(|l| l.update_trackers.merge(adv)) { + layout.update_update_trackers(self.node_id); + assert!(layout.digest() != prev_digest); + return Some(layout.inner().update_trackers.clone()); + } + } + + None + } + + async fn pull_cluster_layout(self: &Arc, peer: Uuid) { + let resp = self + .rpc_helper + .call( + &self.system_endpoint, + peer, + SystemRpc::PullClusterLayout, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await; + if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp { + if let Err(e) = self.handle_advertise_cluster_layout(&layout).await { + warn!("In pull_cluster_layout: {}", e); + } + } + } + + async fn pull_cluster_layout_trackers(self: &Arc, peer: Uuid) { + let resp = self + .rpc_helper + .call( + &self.system_endpoint, + peer, + SystemRpc::PullClusterLayoutTrackers, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await; + if let Ok(SystemRpc::AdvertiseClusterLayoutTrackers(trackers)) = resp { + if let Err(e) = self + .handle_advertise_cluster_layout_trackers(&trackers) + .await + { + warn!("In pull_cluster_layout_trackers: {}", e); + } + } + } + + /// Save cluster layout data to disk + async fn save_cluster_layout(&self) { + let layout = self.layout.read().unwrap().inner().clone(); + if let Err(e) = self.persist_cluster_layout.save_async(&layout).await { + error!("Failed to save cluster_layout: {}", e); + } + } + + fn broadcast_update(self: &Arc, rpc: SystemRpc) { + tokio::spawn({ + let this = self.clone(); + async move { + if let Err(e) = this + .rpc_helper + .broadcast( + &this.system_endpoint, + rpc, + RequestStrategy::with_priority(PRIO_HIGH), + ) + .await + { + warn!("Error while broadcasting new cluster layout: {}", e); + } + } + }); + } + + // ---- RPC HANDLERS ---- + + pub(crate) fn handle_advertise_status(self: &Arc, from: Uuid, remote: &RpcLayoutDigest) { + let local = self.layout().digest(); + if remote.current_version > local.current_version + || remote.active_versions != local.active_versions + || remote.staging_hash != local.staging_hash + { + tokio::spawn({ + let this = self.clone(); + async move { this.pull_cluster_layout(from).await } + }); + } else if remote.trackers_hash != local.trackers_hash { + tokio::spawn({ + let this = self.clone(); + async move { this.pull_cluster_layout_trackers(from).await } + }); + } + } + + pub(crate) fn handle_pull_cluster_layout(&self) -> SystemRpc { + let layout = self.layout.read().unwrap().inner().clone(); + SystemRpc::AdvertiseClusterLayout(layout) + } + + pub(crate) fn handle_pull_cluster_layout_trackers(&self) -> SystemRpc { + let layout = self.layout.read().unwrap(); + SystemRpc::AdvertiseClusterLayoutTrackers(layout.inner().update_trackers.clone()) + } + + pub(crate) async fn handle_advertise_cluster_layout( + self: &Arc, + adv: &LayoutHistory, + ) -> Result { + debug!( + "handle_advertise_cluster_layout: {} versions, last={}, trackers={:?}", + adv.versions.len(), + adv.current().version, + adv.update_trackers + ); + + if adv.current().replication_factor != self.replication_factor.replication_factor() { + let msg = format!( + "Received a cluster layout from another node with replication factor {}, which is different from what we have in our configuration ({}). Discarding the cluster layout we received.", + adv.current().replication_factor, + self.replication_factor.replication_factor() + ); + error!("{}", msg); + return Err(Error::Message(msg)); + } + + if let Some(new_layout) = self.merge_layout(adv) { + debug!("handle_advertise_cluster_layout: some changes were added to the current stuff"); + + self.change_notify.notify_waiters(); + self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout)); + self.save_cluster_layout().await; + } + + Ok(SystemRpc::Ok) + } + + pub(crate) async fn handle_advertise_cluster_layout_trackers( + self: &Arc, + trackers: &UpdateTrackers, + ) -> Result { + debug!("handle_advertise_cluster_layout_trackers: {:?}", trackers); + + if let Some(new_trackers) = self.merge_layout_trackers(trackers) { + self.change_notify.notify_waiters(); + self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers)); + self.save_cluster_layout().await; + } + + Ok(SystemRpc::Ok) + } +} + +// ---- ack lock ---- + +pub struct WriteLock { + layout_version: u64, + layout_manager: Arc, + value: T, +} + +impl WriteLock { + fn new(version: u64, layout_manager: &Arc, value: T) -> Self { + Self { + layout_version: version, + layout_manager: layout_manager.clone(), + value, + } + } +} + +impl AsRef for WriteLock { + fn as_ref(&self) -> &T { + &self.value + } +} + +impl AsMut for WriteLock { + fn as_mut(&mut self) -> &mut T { + &mut self.value + } +} + +impl Drop for WriteLock { + fn drop(&mut self) { + let layout = self.layout_manager.layout(); // acquire read lock + if let Some(counter) = layout.ack_lock.get(&self.layout_version) { + let prev_lock = counter.fetch_sub(1, Ordering::Relaxed); + if prev_lock == 1 && layout.current().version > self.layout_version { + drop(layout); // release read lock, write lock will be acquired + self.layout_manager.ack_new_version(); + } + } else { + error!("Could not find ack lock counter for layout version {}. This probably indicates a bug in Garage.", self.layout_version); + } + } +} diff --git a/src/rpc/layout/mod.rs b/src/rpc/layout/mod.rs new file mode 100644 index 00000000..ce21a524 --- /dev/null +++ b/src/rpc/layout/mod.rs @@ -0,0 +1,478 @@ +use std::fmt; + +use bytesize::ByteSize; + +use garage_util::crdt::{AutoCrdt, Crdt}; +use garage_util::data::Uuid; + +mod graph_algo; +mod helper; +mod history; +mod version; + +#[cfg(test)] +mod test; + +pub mod manager; + +// ---- re-exports ---- + +pub use helper::{LayoutHelper, RpcLayoutDigest, SyncLayoutDigest}; +pub use manager::WriteLock; +pub use version::*; + +// ---- defines: partitions ---- + +/// A partition id, which is stored on 16 bits +/// i.e. we have up to 2**16 partitions. +/// (in practice we have exactly 2**PARTITION_BITS partitions) +pub type Partition = u16; + +// TODO: make this constant parametrizable in the config file +// For deployments with many nodes it might make sense to bump +// it up to 10. +// Maximum value : 16 +/// How many bits from the hash are used to make partitions. Higher numbers means more fairness in +/// presence of numerous nodes, but exponentially bigger ring. Max 16 +pub const PARTITION_BITS: usize = 8; + +const NB_PARTITIONS: usize = 1usize << PARTITION_BITS; + +// ---- defines: nodes ---- + +// Type to store compactly the id of a node in the system +// Change this to u16 the day we want to have more than 256 nodes in a cluster +pub type CompactNodeType = u8; +pub const MAX_NODE_NUMBER: usize = 256; + +// ======== actual data structures for the layout data ======== +// ======== that is persisted to disk ======== +// some small utility impls are at the end of this file, +// but most of the code that actually computes stuff is in +// version.rs, history.rs and helper.rs + +mod v08 { + use crate::layout::CompactNodeType; + use garage_util::crdt::LwwMap; + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + + /// The layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct ClusterLayout { + pub version: u64, + + pub replication_factor: usize, + pub roles: LwwMap, + + // see comments in v010::ClusterLayout + pub node_id_vec: Vec, + #[serde(with = "serde_bytes")] + pub ring_assignation_data: Vec, + + /// Role changes which are staged for the next version of the layout + pub staging: LwwMap, + pub staging_hash: Hash, + } + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct NodeRoleV(pub Option); + + /// The user-assigned roles of cluster nodes + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] + pub struct NodeRole { + /// Datacenter at which this entry belong. This information is used to + /// perform a better geodistribution + pub zone: String, + /// The capacity of the node + /// If this is set to None, the node does not participate in storing data for the system + /// and is only active as an API gateway to other nodes + pub capacity: Option, + /// A set of tags to recognize the node + pub tags: Vec, + } + + impl garage_util::migrate::InitialFormat for ClusterLayout {} +} + +mod v09 { + use super::v08; + use crate::layout::CompactNodeType; + use garage_util::crdt::{Lww, LwwMap}; + use garage_util::data::{Hash, Uuid}; + use serde::{Deserialize, Serialize}; + pub use v08::{NodeRole, NodeRoleV}; + + /// The layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct ClusterLayout { + pub version: u64, + + pub replication_factor: usize, + + /// This attribute is only used to retain the previously computed partition size, + /// to know to what extent does it change with the layout update. + pub partition_size: u64, + /// Parameters used to compute the assignment currently given by + /// ring_assignment_data + pub parameters: LayoutParameters, + + pub roles: LwwMap, + + // see comments in v010::ClusterLayout + pub node_id_vec: Vec, + #[serde(with = "serde_bytes")] + pub ring_assignment_data: Vec, + + /// Parameters to be used in the next partition assignment computation. + pub staging_parameters: Lww, + /// Role changes which are staged for the next version of the layout + pub staging_roles: LwwMap, + pub staging_hash: Hash, + } + + /// This struct is used to set the parameters to be used in the assignment computation + /// algorithm. It is stored as a Crdt. + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub struct LayoutParameters { + pub zone_redundancy: ZoneRedundancy, + } + + /// Zone redundancy: if set to AtLeast(x), the layout calculation will aim to store copies + /// of each partition on at least that number of different zones. + /// Otherwise, copies will be stored on the maximum possible number of zones. + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)] + pub enum ZoneRedundancy { + AtLeast(usize), + Maximum, + } + + impl garage_util::migrate::Migrate for ClusterLayout { + const VERSION_MARKER: &'static [u8] = b"G09layout"; + + type Previous = v08::ClusterLayout; + + fn migrate(previous: Self::Previous) -> Self { + use itertools::Itertools; + + // In the old layout, capacities are in an arbitrary unit, + // but in the new layout they are in bytes. + // Here we arbitrarily multiply everything by 1G, + // such that 1 old capacity unit = 1GB in the new units. + // This is totally arbitrary and won't work for most users. + let cap_mul = 1024 * 1024 * 1024; + let roles = multiply_all_capacities(previous.roles, cap_mul); + let staging_roles = multiply_all_capacities(previous.staging, cap_mul); + let node_id_vec = previous.node_id_vec; + + // Determine partition size + let mut tmp = previous.ring_assignation_data.clone(); + tmp.sort(); + let partition_size = tmp + .into_iter() + .dedup_with_count() + .map(|(npart, node)| { + roles + .get(&node_id_vec[node as usize]) + .and_then(|p| p.0.as_ref().and_then(|r| r.capacity)) + .unwrap_or(0) / npart as u64 + }) + .min() + .unwrap_or(0); + + // By default, zone_redundancy is maximum possible value + let parameters = LayoutParameters { + zone_redundancy: ZoneRedundancy::Maximum, + }; + + Self { + version: previous.version, + replication_factor: previous.replication_factor, + partition_size, + parameters, + roles, + node_id_vec, + ring_assignment_data: previous.ring_assignation_data, + staging_parameters: Lww::new(parameters), + staging_roles, + staging_hash: [0u8; 32].into(), // will be set in the next migration + } + } + } + + fn multiply_all_capacities( + old_roles: LwwMap, + mul: u64, + ) -> LwwMap { + let mut new_roles = LwwMap::new(); + for (node, ts, role) in old_roles.items() { + let mut role = role.clone(); + if let NodeRoleV(Some(NodeRole { + capacity: Some(ref mut cap), + .. + })) = role + { + *cap *= mul; + } + new_roles.merge_raw(node, *ts, &role); + } + new_roles + } +} + +mod v010 { + use super::v09; + use crate::layout::CompactNodeType; + use garage_util::crdt::{Lww, LwwMap}; + use garage_util::data::Uuid; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeMap; + pub use v09::{LayoutParameters, NodeRole, NodeRoleV, ZoneRedundancy}; + + /// Number of old (non-live) versions to keep, see LayoutHistory::old_versions + pub const OLD_VERSION_COUNT: usize = 5; + + /// The history of cluster layouts, with trackers to keep a record + /// of which nodes are up-to-date to current cluster data + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutHistory { + /// The versions currently in use in the cluster + pub versions: Vec, + /// At most 5 of the previous versions, not used by the garage_table + /// module, but useful for the garage_block module to find data blocks + /// that have not yet been moved + pub old_versions: Vec, + + /// Update trackers + pub update_trackers: UpdateTrackers, + + /// Staged changes for the next version + pub staging: Lww, + } + + /// A version of the layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutVersion { + /// The number of this version + pub version: u64, + + /// Roles assigned to nodes in this version + pub roles: LwwMap, + /// Parameters used to compute the assignment currently given by + /// ring_assignment_data + pub parameters: LayoutParameters, + + /// The number of replicas for each data partition + pub replication_factor: usize, + /// This attribute is only used to retain the previously computed partition size, + /// to know to what extent does it change with the layout update. + pub partition_size: u64, + + /// node_id_vec: a vector of node IDs with a role assigned + /// in the system (this includes gateway nodes). + /// The order here is different than the vec stored by `roles`, because: + /// 1. non-gateway nodes are first so that they have lower numbers + /// 2. nodes that don't have a role are excluded (but they need to + /// stay in the CRDT as tombstones) + pub node_id_vec: Vec, + /// number of non-gateway nodes, which are the first ids in node_id_vec + pub nongateway_node_count: usize, + /// The assignation of data partitions to nodes, the values + /// are indices in node_id_vec + #[serde(with = "serde_bytes")] + pub ring_assignment_data: Vec, + } + + /// The staged changes for the next layout version + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] + pub struct LayoutStaging { + /// Parameters to be used in the next partition assignment computation. + pub parameters: Lww, + /// Role changes which are staged for the next version of the layout + pub roles: LwwMap, + } + + /// The tracker of acknowlegments and data syncs around the cluster + #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq)] + pub struct UpdateTrackers { + /// The highest layout version number each node has ack'ed + pub ack_map: UpdateTracker, + /// The highest layout version number each node has synced data for + pub sync_map: UpdateTracker, + /// The highest layout version number each node has + /// ack'ed that all other nodes have synced data for + pub sync_ack_map: UpdateTracker, + } + + /// Generic update tracker struct + #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq)] + pub struct UpdateTracker(pub BTreeMap); + + impl garage_util::migrate::Migrate for LayoutHistory { + const VERSION_MARKER: &'static [u8] = b"G010lh"; + + type Previous = v09::ClusterLayout; + + fn migrate(previous: Self::Previous) -> Self { + let nongateway_node_count = previous + .node_id_vec + .iter() + .enumerate() + .filter(|(_, uuid)| { + let role = previous.roles.get(uuid); + matches!(role, Some(NodeRoleV(Some(role))) if role.capacity.is_some()) + }) + .map(|(i, _)| i + 1) + .max() + .unwrap_or(0); + + let version = LayoutVersion { + version: previous.version, + replication_factor: previous.replication_factor, + partition_size: previous.partition_size, + parameters: previous.parameters, + roles: previous.roles, + node_id_vec: previous.node_id_vec, + nongateway_node_count, + ring_assignment_data: previous.ring_assignment_data, + }; + let update_tracker = UpdateTracker( + version + .nongateway_nodes() + .iter() + .copied() + .map(|x| (x, version.version)) + .collect::>(), + ); + let staging = LayoutStaging { + parameters: previous.staging_parameters, + roles: previous.staging_roles, + }; + Self { + versions: vec![version], + old_versions: vec![], + update_trackers: UpdateTrackers { + ack_map: update_tracker.clone(), + sync_map: update_tracker.clone(), + sync_ack_map: update_tracker, + }, + staging: Lww::raw(previous.version, staging), + } + } + } +} + +pub use v010::*; + +// ---- utility functions ---- + +impl AutoCrdt for LayoutParameters { + const WARN_IF_DIFFERENT: bool = true; +} + +impl AutoCrdt for NodeRoleV { + const WARN_IF_DIFFERENT: bool = true; +} + +impl Crdt for LayoutStaging { + fn merge(&mut self, other: &LayoutStaging) { + self.parameters.merge(&other.parameters); + self.roles.merge(&other.roles); + } +} + +impl NodeRole { + pub fn capacity_string(&self) -> String { + match self.capacity { + Some(c) => ByteSize::b(c).to_string_as(false), + None => "gateway".to_string(), + } + } + + pub fn tags_string(&self) -> String { + self.tags.join(",") + } +} + +impl fmt::Display for ZoneRedundancy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ZoneRedundancy::Maximum => write!(f, "maximum"), + ZoneRedundancy::AtLeast(x) => write!(f, "{}", x), + } + } +} + +impl core::str::FromStr for ZoneRedundancy { + type Err = &'static str; + fn from_str(s: &str) -> Result { + match s { + "none" | "max" | "maximum" => Ok(ZoneRedundancy::Maximum), + x => { + let v = x + .parse::() + .map_err(|_| "zone redundancy must be 'none'/'max' or an integer")?; + Ok(ZoneRedundancy::AtLeast(v)) + } + } + } +} + +impl UpdateTracker { + fn merge(&mut self, other: &UpdateTracker) -> bool { + let mut changed = false; + for (k, v) in other.0.iter() { + if let Some(v_mut) = self.0.get_mut(k) { + if *v > *v_mut { + *v_mut = *v; + changed = true; + } + } else { + self.0.insert(*k, *v); + changed = true; + } + } + changed + } + + /// This bumps the update tracker for a given node up to the specified value. + /// This has potential impacts on the correctness of Garage and should only + /// be used in very specific circumstances. + pub fn set_max(&mut self, peer: Uuid, value: u64) -> bool { + match self.0.get_mut(&peer) { + Some(e) if *e < value => { + *e = value; + true + } + None => { + self.0.insert(peer, value); + true + } + _ => false, + } + } + + pub fn min_among(&self, storage_nodes: &[Uuid], min_version: u64) -> u64 { + storage_nodes + .iter() + .map(|x| self.get(x, min_version)) + .min() + .unwrap_or(min_version) + } + + pub fn get(&self, node: &Uuid, min_version: u64) -> u64 { + self.0.get(node).copied().unwrap_or(min_version) + } +} + +impl UpdateTrackers { + pub(crate) fn merge(&mut self, other: &UpdateTrackers) -> bool { + let c1 = self.ack_map.merge(&other.ack_map); + let c2 = self.sync_map.merge(&other.sync_map); + let c3 = self.sync_ack_map.merge(&other.sync_ack_map); + c1 || c2 || c3 + } +} diff --git a/src/rpc/layout/test.rs b/src/rpc/layout/test.rs new file mode 100644 index 00000000..5462160b --- /dev/null +++ b/src/rpc/layout/test.rs @@ -0,0 +1,158 @@ +use std::cmp::min; +use std::collections::HashMap; + +use garage_util::crdt::Crdt; +use garage_util::error::*; + +use crate::layout::*; +use crate::replication_mode::ReplicationFactor; + +// This function checks that the partition size S computed is at least better than the +// one given by a very naive algorithm. To do so, we try to run the naive algorithm +// assuming a partition size of S+1. If we succeed, it means that the optimal assignment +// was not optimal. The naive algorithm is the following : +// - we compute the max number of partitions associated to every node, capped at the +// partition number. It gives the number of tokens of every node. +// - every zone has a number of tokens equal to the sum of the tokens of its nodes. +// - we cycle over the partitions and associate zone tokens while respecting the +// zone redundancy constraint. +// NOTE: the naive algorithm is not optimal. Counter example: +// take nb_partition = 3 ; replication_factor = 5; redundancy = 4; +// number of tokens by zone : (A, 4), (B,1), (C,4), (D, 4), (E, 2) +// With these parameters, the naive algo fails, whereas there is a solution: +// (A,A,C,D,E) , (A,B,C,D,D) (A,C,C,D,E) +fn check_against_naive(cl: &LayoutVersion) -> Result { + let over_size = cl.partition_size + 1; + let mut zone_token = HashMap::::new(); + + let (zones, zone_to_id) = cl.generate_nongateway_zone_ids()?; + + if zones.is_empty() { + return Ok(false); + } + + for z in zones.iter() { + zone_token.insert(z.clone(), 0); + } + for uuid in cl.nongateway_nodes() { + let z = cl.expect_get_node_zone(&uuid); + let c = cl.expect_get_node_capacity(&uuid); + zone_token.insert( + z.to_string(), + zone_token[z] + min(NB_PARTITIONS, (c / over_size) as usize), + ); + } + + // For every partition, we count the number of zone already associated and + // the name of the last zone associated + + let mut id_zone_token = vec![0; zones.len()]; + for (z, t) in zone_token.iter() { + id_zone_token[zone_to_id[z]] = *t; + } + + let mut nb_token = vec![0; NB_PARTITIONS]; + let mut last_zone = vec![zones.len(); NB_PARTITIONS]; + + let mut curr_zone = 0; + + let redundancy = cl.effective_zone_redundancy(); + + for replic in 0..cl.replication_factor { + for p in 0..NB_PARTITIONS { + while id_zone_token[curr_zone] == 0 + || (last_zone[p] == curr_zone + && redundancy - nb_token[p] <= cl.replication_factor - replic) + { + curr_zone += 1; + if curr_zone >= zones.len() { + return Ok(true); + } + } + id_zone_token[curr_zone] -= 1; + if last_zone[p] != curr_zone { + nb_token[p] += 1; + last_zone[p] = curr_zone; + } + } + } + + return Ok(false); +} + +fn show_msg(msg: &Message) { + for s in msg.iter() { + println!("{}", s); + } +} + +fn update_layout( + cl: &mut LayoutHistory, + node_capacity_vec: &[u64], + node_zone_vec: &[&'static str], + zone_redundancy: usize, +) { + let staging = cl.staging.get_mut(); + + for (i, (capacity, zone)) in node_capacity_vec + .iter() + .zip(node_zone_vec.iter()) + .enumerate() + { + let node_id = [i as u8; 32].into(); + + let update = staging.roles.update_mutator( + node_id, + NodeRoleV(Some(NodeRole { + zone: zone.to_string(), + capacity: Some(*capacity), + tags: (vec![]), + })), + ); + staging.roles.merge(&update); + } + staging.parameters.update(LayoutParameters { + zone_redundancy: ZoneRedundancy::AtLeast(zone_redundancy), + }); +} + +#[test] +fn test_assignment() { + let mut node_capacity_vec = vec![4000, 1000, 2000]; + let mut node_zone_vec = vec!["A", "B", "C"]; + + let mut cl = LayoutHistory::new(ReplicationFactor::new(3).unwrap()); + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 3); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![4000, 1000, 1000, 3000, 1000, 1000, 2000, 10000, 2000]; + node_zone_vec = vec!["A", "B", "C", "C", "C", "B", "G", "H", "I"]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 2); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![4000, 1000, 2000, 7000, 1000, 1000, 2000, 10000, 2000]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 3); + let v = cl.current().version; + let (mut cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); + + node_capacity_vec = vec![ + 4000000, 4000000, 2000000, 7000000, 1000000, 9000000, 2000000, 10000, 2000000, + ]; + update_layout(&mut cl, &node_capacity_vec, &node_zone_vec, 1); + let v = cl.current().version; + let (cl, msg) = cl.apply_staged_changes(Some(v + 1)).unwrap(); + show_msg(&msg); + assert_eq!(cl.check(), Ok(())); + assert!(check_against_naive(cl.current()).unwrap()); +} diff --git a/src/rpc/layout/version.rs b/src/rpc/layout/version.rs new file mode 100644 index 00000000..a02fce89 --- /dev/null +++ b/src/rpc/layout/version.rs @@ -0,0 +1,852 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::convert::TryInto; + +use bytesize::ByteSize; +use itertools::Itertools; + +use garage_util::crdt::{Crdt, LwwMap}; +use garage_util::data::*; +use garage_util::error::*; + +use super::graph_algo::*; +use super::*; + +// The Message type will be used to collect information on the algorithm. +pub type Message = Vec; + +impl LayoutVersion { + pub fn new(replication_factor: usize) -> Self { + // We set the default zone redundancy to be Maximum, meaning that the maximum + // possible value will be used depending on the cluster topology + let parameters = LayoutParameters { + zone_redundancy: ZoneRedundancy::Maximum, + }; + + LayoutVersion { + version: 0, + replication_factor, + partition_size: 0, + roles: LwwMap::new(), + node_id_vec: Vec::new(), + nongateway_node_count: 0, + ring_assignment_data: Vec::new(), + parameters, + } + } + + // ===================== accessors ====================== + + /// Returns a list of IDs of nodes that have a role in this + /// version of the cluster layout, including gateway nodes + pub fn all_nodes(&self) -> &[Uuid] { + &self.node_id_vec[..] + } + + /// Returns a list of IDs of nodes that have a storage capacity + /// assigned in this version of the cluster layout + pub fn nongateway_nodes(&self) -> &[Uuid] { + &self.node_id_vec[..self.nongateway_node_count] + } + + /// Returns the role of a node in the layout, if it has one + pub fn node_role(&self, node: &Uuid) -> Option<&NodeRole> { + match self.roles.get(node) { + Some(NodeRoleV(Some(v))) => Some(v), + _ => None, + } + } + + /// Returns the capacity of a node in the layout, if it has one + pub fn get_node_capacity(&self, uuid: &Uuid) -> Option { + match self.node_role(uuid) { + Some(NodeRole { + capacity: Some(cap), + zone: _, + tags: _, + }) => Some(*cap), + _ => None, + } + } + + /// Given a node uuids, this function returns the label of its zone if it has one + pub fn get_node_zone(&self, uuid: &Uuid) -> Option<&str> { + match self.node_role(uuid) { + Some(role) => Some(&role.zone), + _ => None, + } + } + + /// Returns the number of partitions associated to this node in the ring + pub fn get_node_usage(&self, uuid: &Uuid) -> Result { + for (i, id) in self.node_id_vec.iter().enumerate() { + if id == uuid { + let mut count = 0; + for nod in self.ring_assignment_data.iter() { + if i as u8 == *nod { + count += 1 + } + } + return Ok(count); + } + } + Err(Error::Message( + "The Uuid does not correspond to a node present in the \ + cluster or this node does not have a positive capacity." + .into(), + )) + } + + /// Get the partition in which data would fall on + pub fn partition_of(&self, position: &Hash) -> Partition { + let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); + top >> (16 - PARTITION_BITS) + } + + /// Get the list of partitions and the first hash of a partition key that would fall in it + pub fn partitions(&self) -> impl Iterator + '_ { + (0..(1 << PARTITION_BITS)).map(|i| { + let top = (i as u16) << (16 - PARTITION_BITS); + let mut location = [0u8; 32]; + location[..2].copy_from_slice(&u16::to_be_bytes(top)[..]); + (i as u16, Hash::from(location)) + }) + } + + /// Return the n servers in which data for this hash should be replicated + pub fn nodes_of(&self, position: &Hash, n: usize) -> impl Iterator + '_ { + assert_eq!(n, self.replication_factor); + + let data = &self.ring_assignment_data; + + let partition_nodes = if data.len() == self.replication_factor * (1 << PARTITION_BITS) { + let partition_idx = self.partition_of(position) as usize; + let partition_start = partition_idx * self.replication_factor; + let partition_end = (partition_idx + 1) * self.replication_factor; + &data[partition_start..partition_end] + } else { + warn!("Ring not yet ready, read/writes will be lost!"); + &[] + }; + + partition_nodes + .iter() + .map(move |i| self.node_id_vec[*i as usize]) + } + + // ===================== internal information extractors ====================== + + pub(crate) fn expect_get_node_capacity(&self, uuid: &Uuid) -> u64 { + self.get_node_capacity(uuid) + .expect("non-gateway node with zero capacity") + } + + pub(crate) fn expect_get_node_zone(&self, uuid: &Uuid) -> &str { + self.get_node_zone(uuid).expect("node without a zone") + } + + /// Returns the sum of capacities of non gateway nodes in the cluster + fn get_total_capacity(&self) -> u64 { + let mut total_capacity = 0; + for uuid in self.nongateway_nodes() { + total_capacity += self.expect_get_node_capacity(uuid); + } + total_capacity + } + + /// Returns the effective value of the zone_redundancy parameter + pub(crate) fn effective_zone_redundancy(&self) -> usize { + match self.parameters.zone_redundancy { + ZoneRedundancy::AtLeast(v) => v, + ZoneRedundancy::Maximum => { + let n_zones = self + .roles + .items() + .iter() + .filter_map(|(_, _, role)| role.0.as_ref().map(|x| x.zone.as_str())) + .collect::>() + .len(); + std::cmp::min(n_zones, self.replication_factor) + } + } + } + + /// Check a cluster layout for internal consistency + /// (assignment, roles, parameters, partition size) + /// returns true if consistent, false if error + pub fn check(&self) -> Result<(), String> { + // Check that the assignment data has the correct length + let expected_assignment_data_len = (1 << PARTITION_BITS) * self.replication_factor; + if self.ring_assignment_data.len() != expected_assignment_data_len { + return Err(format!( + "ring_assignment_data has incorrect length {} instead of {}", + self.ring_assignment_data.len(), + expected_assignment_data_len + )); + } + + // Check that node_id_vec contains the correct list of nodes + let mut expected_nodes = self + .roles + .items() + .iter() + .filter(|(_, _, v)| v.0.is_some()) + .map(|(id, _, _)| *id) + .collect::>(); + expected_nodes.sort(); + let mut node_id_vec = self.node_id_vec.clone(); + node_id_vec.sort(); + if expected_nodes != node_id_vec { + return Err(format!("node_id_vec does not contain the correct set of nodes\nnode_id_vec: {:?}\nexpected: {:?}", node_id_vec, expected_nodes)); + } + + // Check that the assigned nodes are correct identifiers + // of nodes that are assigned a role + // and that role is not the role of a gateway nodes + for x in self.ring_assignment_data.iter() { + if *x as usize >= self.node_id_vec.len() { + return Err(format!( + "ring_assignment_data contains invalid node id {}", + *x + )); + } + let node = self.node_id_vec[*x as usize]; + match self.roles.get(&node) { + Some(NodeRoleV(Some(x))) if x.capacity.is_some() => (), + _ => return Err("ring_assignment_data contains id of a gateway node".into()), + } + } + + // Check that every partition is associated to distinct nodes + let zone_redundancy = self.effective_zone_redundancy(); + let rf = self.replication_factor; + for p in 0..(1 << PARTITION_BITS) { + let nodes_of_p = self.ring_assignment_data[rf * p..rf * (p + 1)].to_vec(); + if nodes_of_p.iter().unique().count() != rf { + return Err(format!("partition does not contain {} unique node ids", rf)); + } + // Check that every partition is spread over at least zone_redundancy zones. + let zones_of_p = nodes_of_p + .iter() + .map(|n| self.expect_get_node_zone(&self.node_id_vec[*n as usize])) + .collect::>(); + if zones_of_p.iter().unique().count() < zone_redundancy { + return Err(format!( + "nodes of partition are in less than {} distinct zones", + zone_redundancy + )); + } + } + + // Check that the nodes capacities is consistent with the stored partitions + let mut node_usage = vec![0; MAX_NODE_NUMBER]; + for n in self.ring_assignment_data.iter() { + node_usage[*n as usize] += 1; + } + for (n, usage) in node_usage.iter().enumerate() { + if *usage > 0 { + let uuid = self.node_id_vec[n]; + let partusage = usage * self.partition_size; + let nodecap = self.expect_get_node_capacity(&uuid); + if partusage > nodecap { + return Err(format!( + "node usage ({}) is bigger than node capacity ({})", + usage * self.partition_size, + nodecap + )); + } + } + } + + // Check that the partition size stored is the one computed by the asignation + // algorithm. + let cl2 = self.clone(); + let (_, zone_to_id) = cl2.generate_nongateway_zone_ids().unwrap(); + match cl2.compute_optimal_partition_size(&zone_to_id, zone_redundancy) { + Ok(s) if s != self.partition_size => { + return Err(format!( + "partition_size ({}) is different than optimal value ({})", + self.partition_size, s + )) + } + Err(e) => return Err(format!("could not calculate optimal partition size: {}", e)), + _ => (), + } + + Ok(()) + } + + // ================== updates to layout, internals =================== + + pub(crate) fn calculate_next_version( + mut self, + staging: &LayoutStaging, + ) -> Result<(Self, Message), Error> { + self.version += 1; + + self.roles.merge(&staging.roles); + self.roles.retain(|(_, _, v)| v.0.is_some()); + self.parameters = *staging.parameters.get(); + + let msg = self.calculate_partition_assignment()?; + + Ok((self, msg)) + } + + /// This function calculates a new partition-to-node assignment. + /// The computed assignment respects the node replication factor + /// and the zone redundancy parameter It maximizes the capacity of a + /// partition (assuming all partitions have the same size). + /// Among such optimal assignment, it minimizes the distance to + /// the former assignment (if any) to minimize the amount of + /// data to be moved. + /// Staged role changes must be merged with nodes roles before calling this function, + /// hence it must only be called from apply_staged_changes() and hence is not public. + fn calculate_partition_assignment(&mut self) -> Result { + // We update the node ids, since the node role list might have changed with the + // changes in the layout. We retrieve the old_assignment reframed with new ids + let old_assignment_opt = self.update_node_id_vec()?; + + let zone_redundancy = self.effective_zone_redundancy(); + + let mut msg = Message::new(); + msg.push("==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====".into()); + msg.push("".into()); + msg.push(format!( + "Partitions are \ + replicated {} times on at least {} distinct zones.", + self.replication_factor, zone_redundancy + )); + + // We generate for once numerical ids for the zones of non gateway nodes, + // to use them as indices in the flow graphs. + let (id_to_zone, zone_to_id) = self.generate_nongateway_zone_ids()?; + + if self.nongateway_nodes().len() < self.replication_factor { + return Err(Error::Message(format!( + "The number of nodes with positive \ + capacity ({}) is smaller than the replication factor ({}).", + self.nongateway_nodes().len(), + self.replication_factor + ))); + } + if id_to_zone.len() < zone_redundancy { + return Err(Error::Message(format!( + "The number of zones with non-gateway \ + nodes ({}) is smaller than the redundancy parameter ({})", + id_to_zone.len(), + zone_redundancy + ))); + } + + // We compute the optimal partition size + // Capacities should be given in a unit so that partition size is at least 100. + // In this case, integer rounding plays a marginal role in the percentages of + // optimality. + let partition_size = self.compute_optimal_partition_size(&zone_to_id, zone_redundancy)?; + + msg.push("".into()); + if old_assignment_opt.is_some() { + msg.push(format!( + "Optimal partition size: {} ({} in previous layout)", + ByteSize::b(partition_size).to_string_as(false), + ByteSize::b(self.partition_size).to_string_as(false) + )); + } else { + msg.push(format!( + "Optimal partition size: {}", + ByteSize::b(partition_size).to_string_as(false) + )); + } + // We write the partition size. + self.partition_size = partition_size; + + if partition_size < 100 { + msg.push( + "WARNING: The partition size is low (< 100), make sure the capacities of your nodes are correct and are of at least a few MB" + .into(), + ); + } + + // We compute a first flow/assignment that is heuristically close to the previous + // assignment + let mut gflow = + self.compute_candidate_assignment(&zone_to_id, &old_assignment_opt, zone_redundancy)?; + if let Some(assoc) = &old_assignment_opt { + // We minimize the distance to the previous assignment. + self.minimize_rebalance_load(&mut gflow, &zone_to_id, assoc)?; + } + + // We display statistics of the computation + msg.extend(self.output_stat(&gflow, &old_assignment_opt, &zone_to_id, &id_to_zone)?); + + // We update the layout structure + self.update_ring_from_flow(id_to_zone.len(), &gflow)?; + + if let Err(e) = self.check() { + return Err(Error::Message( + format!("Layout check returned an error: {}\nOriginal result of computation: <<<<\n{}\n>>>>", e, msg.join("\n")) + )); + } + + Ok(msg) + } + + /// The LwwMap of node roles might have changed. This function updates the node_id_vec + /// and returns the assignment given by ring, with the new indices of the nodes, and + /// None if the node is not present anymore. + /// We work with the assumption that only this function and calculate_new_assignment + /// do modify assignment_ring and node_id_vec. + fn update_node_id_vec(&mut self) -> Result>>, Error> { + // (1) We compute the new node list + // Non gateway nodes should be coded on 8bits, hence they must be first in the list + // We build the new node ids + let new_non_gateway_nodes: Vec = self + .roles + .items() + .iter() + .filter(|(_, _, v)| matches!(&v.0, Some(r) if r.capacity.is_some())) + .map(|(k, _, _)| *k) + .collect(); + + if new_non_gateway_nodes.len() > MAX_NODE_NUMBER { + return Err(Error::Message(format!( + "There are more than {} non-gateway nodes in the new \ + layout. This is not allowed.", + MAX_NODE_NUMBER + ))); + } + + let new_gateway_nodes: Vec = self + .roles + .items() + .iter() + .filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_none())) + .map(|(k, _, _)| *k) + .collect(); + + let old_node_id_vec = std::mem::take(&mut self.node_id_vec); + + self.nongateway_node_count = new_non_gateway_nodes.len(); + self.node_id_vec.clear(); + self.node_id_vec.extend(new_non_gateway_nodes); + self.node_id_vec.extend(new_gateway_nodes); + + let new_node_id_vec = &self.node_id_vec; + + // (2) We retrieve the old association + // We rewrite the old association with the new indices. We only consider partition + // to node assignments where the node is still in use. + if self.ring_assignment_data.is_empty() { + // This is a new association + return Ok(None); + } + + if self.ring_assignment_data.len() != NB_PARTITIONS * self.replication_factor { + return Err(Error::Message( + "The old assignment does not have a size corresponding to \ + the old replication factor or the number of partitions." + .into(), + )); + } + + // We build a translation table between the uuid and new ids + let mut uuid_to_new_id = HashMap::::new(); + + // We add the indices of only the new non-gateway nodes that can be used in the + // association ring + for (i, uuid) in new_node_id_vec.iter().enumerate() { + uuid_to_new_id.insert(*uuid, i); + } + + let mut old_assignment = vec![Vec::::new(); NB_PARTITIONS]; + let rf = self.replication_factor; + + for (p, old_assign_p) in old_assignment.iter_mut().enumerate() { + for old_id in &self.ring_assignment_data[p * rf..(p + 1) * rf] { + let uuid = old_node_id_vec[*old_id as usize]; + if uuid_to_new_id.contains_key(&uuid) { + old_assign_p.push(uuid_to_new_id[&uuid]); + } + } + } + + // We clear the ring assignment data + self.ring_assignment_data = Vec::::new(); + + Ok(Some(old_assignment)) + } + + /// This function generates ids for the zone of the nodes appearing in + /// self.node_id_vec. + pub(crate) fn generate_nongateway_zone_ids( + &self, + ) -> Result<(Vec, HashMap), Error> { + let mut id_to_zone = Vec::::new(); + let mut zone_to_id = HashMap::::new(); + + for uuid in self.nongateway_nodes().iter() { + let r = self.node_role(uuid).unwrap(); + if !zone_to_id.contains_key(&r.zone) && r.capacity.is_some() { + zone_to_id.insert(r.zone.clone(), id_to_zone.len()); + id_to_zone.push(r.zone.clone()); + } + } + Ok((id_to_zone, zone_to_id)) + } + + /// This function computes by dichotomy the largest realizable partition size, given + /// the layout roles and parameters. + fn compute_optimal_partition_size( + &self, + zone_to_id: &HashMap, + zone_redundancy: usize, + ) -> Result { + let empty_set = HashSet::<(usize, usize)>::new(); + let mut g = self.generate_flow_graph(1, zone_to_id, &empty_set, zone_redundancy)?; + g.compute_maximal_flow()?; + if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 { + return Err(Error::Message( + "The storage capacity of the cluster is too small. It is \ + impossible to store partitions of size 1." + .into(), + )); + } + + let mut s_down = 1; + let mut s_up = self.get_total_capacity(); + while s_down + 1 < s_up { + g = self.generate_flow_graph( + (s_down + s_up) / 2, + zone_to_id, + &empty_set, + zone_redundancy, + )?; + g.compute_maximal_flow()?; + if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 { + s_up = (s_down + s_up) / 2; + } else { + s_down = (s_down + s_up) / 2; + } + } + + Ok(s_down) + } + + fn generate_graph_vertices(nb_zones: usize, nb_nodes: usize) -> Vec { + let mut vertices = vec![Vertex::Source, Vertex::Sink]; + for p in 0..NB_PARTITIONS { + vertices.push(Vertex::Pup(p)); + vertices.push(Vertex::Pdown(p)); + for z in 0..nb_zones { + vertices.push(Vertex::PZ(p, z)); + } + } + for n in 0..nb_nodes { + vertices.push(Vertex::N(n)); + } + vertices + } + + /// Generates the graph to compute the maximal flow corresponding to the optimal + /// partition assignment. + /// exclude_assoc is the set of (partition, node) association that we are forbidden + /// to use (hence we do not add the corresponding edge to the graph). This parameter + /// is used to compute a first flow that uses only edges appearing in the previous + /// assignment. This produces a solution that heuristically should be close to the + /// previous one. + fn generate_flow_graph( + &self, + partition_size: u64, + zone_to_id: &HashMap, + exclude_assoc: &HashSet<(usize, usize)>, + zone_redundancy: usize, + ) -> Result, Error> { + let vertices = + LayoutVersion::generate_graph_vertices(zone_to_id.len(), self.nongateway_nodes().len()); + let mut g = Graph::::new(&vertices); + let nb_zones = zone_to_id.len(); + for p in 0..NB_PARTITIONS { + g.add_edge(Vertex::Source, Vertex::Pup(p), zone_redundancy as u64)?; + g.add_edge( + Vertex::Source, + Vertex::Pdown(p), + (self.replication_factor - zone_redundancy) as u64, + )?; + for z in 0..nb_zones { + g.add_edge(Vertex::Pup(p), Vertex::PZ(p, z), 1)?; + g.add_edge( + Vertex::Pdown(p), + Vertex::PZ(p, z), + self.replication_factor as u64, + )?; + } + } + for n in 0..self.nongateway_nodes().len() { + let node_capacity = self.expect_get_node_capacity(&self.node_id_vec[n]); + let node_zone = zone_to_id[self.expect_get_node_zone(&self.node_id_vec[n])]; + g.add_edge(Vertex::N(n), Vertex::Sink, node_capacity / partition_size)?; + for p in 0..NB_PARTITIONS { + if !exclude_assoc.contains(&(p, n)) { + g.add_edge(Vertex::PZ(p, node_zone), Vertex::N(n), 1)?; + } + } + } + Ok(g) + } + + /// This function computes a first optimal assignment (in the form of a flow graph). + fn compute_candidate_assignment( + &self, + zone_to_id: &HashMap, + prev_assign_opt: &Option>>, + zone_redundancy: usize, + ) -> Result, Error> { + // We list the (partition,node) associations that are not used in the + // previous assignment + let mut exclude_edge = HashSet::<(usize, usize)>::new(); + if let Some(prev_assign) = prev_assign_opt { + let nb_nodes = self.nongateway_nodes().len(); + for (p, prev_assign_p) in prev_assign.iter().enumerate() { + for n in 0..nb_nodes { + exclude_edge.insert((p, n)); + } + for n in prev_assign_p.iter() { + exclude_edge.remove(&(p, *n)); + } + } + } + + // We compute the best flow using only the edges used in the previous assignment + let mut g = self.generate_flow_graph( + self.partition_size, + zone_to_id, + &exclude_edge, + zone_redundancy, + )?; + g.compute_maximal_flow()?; + + // We add the excluded edges and compute the maximal flow with the full graph. + // The algorithm is such that it will start with the flow that we just computed + // and find ameliorating paths from that. + for (p, n) in exclude_edge.iter() { + let node_zone = zone_to_id[self.expect_get_node_zone(&self.node_id_vec[*n])]; + g.add_edge(Vertex::PZ(*p, node_zone), Vertex::N(*n), 1)?; + } + g.compute_maximal_flow()?; + Ok(g) + } + + /// This function updates the flow graph gflow to minimize the distance between + /// its corresponding assignment and the previous one + fn minimize_rebalance_load( + &self, + gflow: &mut Graph, + zone_to_id: &HashMap, + prev_assign: &[Vec], + ) -> Result<(), Error> { + // We define a cost function on the edges (pairs of vertices) corresponding + // to the distance between the two assignments. + let mut cost = CostFunction::new(); + for (p, assoc_p) in prev_assign.iter().enumerate() { + for n in assoc_p.iter() { + if let Some(&node_zone) = + zone_to_id.get(self.expect_get_node_zone(&self.node_id_vec[*n])) + { + cost.insert((Vertex::PZ(p, node_zone), Vertex::N(*n)), -1); + } + } + } + + // We compute the maximal length of a simple path in gflow. It is used in the + // Bellman-Ford algorithm in optimize_flow_with_cost to set the number + // of iterations. + let nb_nodes = self.nongateway_nodes().len(); + let path_length = 4 * nb_nodes; + gflow.optimize_flow_with_cost(&cost, path_length)?; + + Ok(()) + } + + /// This function updates the assignment ring from the flow graph. + fn update_ring_from_flow( + &mut self, + nb_zones: usize, + gflow: &Graph, + ) -> Result<(), Error> { + self.ring_assignment_data = Vec::::new(); + for p in 0..NB_PARTITIONS { + for z in 0..nb_zones { + let assoc_vertex = gflow.get_positive_flow_from(Vertex::PZ(p, z))?; + for vertex in assoc_vertex.iter() { + if let Vertex::N(n) = vertex { + self.ring_assignment_data.push((*n).try_into().unwrap()); + } + } + } + } + + if self.ring_assignment_data.len() != NB_PARTITIONS * self.replication_factor { + return Err(Error::Message( + "Critical Error : the association ring we produced does not \ + have the right size." + .into(), + )); + } + Ok(()) + } + + /// This function returns a message summing up the partition repartition of the new + /// layout, and other statistics of the partition assignment computation. + fn output_stat( + &self, + gflow: &Graph, + prev_assign_opt: &Option>>, + zone_to_id: &HashMap, + id_to_zone: &[String], + ) -> Result { + let mut msg = Message::new(); + + let used_cap = self.partition_size * NB_PARTITIONS as u64 * self.replication_factor as u64; + let total_cap = self.get_total_capacity(); + let percent_cap = 100.0 * (used_cap as f32) / (total_cap as f32); + msg.push(format!( + "Usable capacity / total cluster capacity: {} / {} ({:.1} %)", + ByteSize::b(used_cap).to_string_as(false), + ByteSize::b(total_cap).to_string_as(false), + percent_cap + )); + msg.push(format!( + "Effective capacity (replication factor {}): {}", + self.replication_factor, + ByteSize::b(used_cap / self.replication_factor as u64).to_string_as(false) + )); + if percent_cap < 80. { + msg.push("".into()); + msg.push( + "If the percentage is too low, it might be that the \ + cluster topology and redundancy constraints are forcing the use of nodes/zones with small \ + storage capacities." + .into(), + ); + msg.push( + "You might want to move storage capacity between zones or relax the redundancy constraint." + .into(), + ); + msg.push( + "See the detailed statistics below and look for saturated nodes/zones.".into(), + ); + } + + // We define and fill in the following tables + let storing_nodes = self.nongateway_nodes(); + let mut new_partitions = vec![0; storing_nodes.len()]; + let mut stored_partitions = vec![0; storing_nodes.len()]; + + let mut new_partitions_zone = vec![0; id_to_zone.len()]; + let mut stored_partitions_zone = vec![0; id_to_zone.len()]; + + for p in 0..NB_PARTITIONS { + for z in 0..id_to_zone.len() { + let pz_nodes = gflow.get_positive_flow_from(Vertex::PZ(p, z))?; + if !pz_nodes.is_empty() { + stored_partitions_zone[z] += 1; + if let Some(prev_assign) = prev_assign_opt { + let mut old_zones_of_p = Vec::::new(); + for n in prev_assign[p].iter() { + if let Some(&zone_id) = + zone_to_id.get(self.expect_get_node_zone(&self.node_id_vec[*n])) + { + old_zones_of_p.push(zone_id); + } + } + if !old_zones_of_p.contains(&z) { + new_partitions_zone[z] += 1; + } + } + } + for vert in pz_nodes.iter() { + if let Vertex::N(n) = *vert { + stored_partitions[n] += 1; + if let Some(prev_assign) = prev_assign_opt { + if !prev_assign[p].contains(&n) { + new_partitions[n] += 1; + } + } + } + } + } + } + + if prev_assign_opt.is_none() { + new_partitions = stored_partitions.clone(); + //new_partitions_zone = stored_partitions_zone.clone(); + } + + // We display the statistics + + msg.push("".into()); + if prev_assign_opt.is_some() { + let total_new_partitions: usize = new_partitions.iter().sum(); + msg.push(format!( + "A total of {} new copies of partitions need to be \ + transferred.", + total_new_partitions + )); + msg.push("".into()); + } + + let mut table = vec![]; + for z in 0..id_to_zone.len() { + let mut nodes_of_z = Vec::::new(); + for n in 0..storing_nodes.len() { + if self.expect_get_node_zone(&self.node_id_vec[n]) == id_to_zone[z] { + nodes_of_z.push(n); + } + } + let replicated_partitions: usize = + nodes_of_z.iter().map(|n| stored_partitions[*n]).sum(); + table.push(format!( + "{}\tTags\tPartitions\tCapacity\tUsable capacity", + id_to_zone[z] + )); + + let available_cap_z: u64 = self.partition_size * replicated_partitions as u64; + let mut total_cap_z = 0; + for n in nodes_of_z.iter() { + total_cap_z += self.expect_get_node_capacity(&self.node_id_vec[*n]); + } + let percent_cap_z = 100.0 * (available_cap_z as f32) / (total_cap_z as f32); + + for n in nodes_of_z.iter() { + let available_cap_n = stored_partitions[*n] as u64 * self.partition_size; + let total_cap_n = self.expect_get_node_capacity(&self.node_id_vec[*n]); + let tags_n = (self.node_role(&self.node_id_vec[*n]).ok_or(""))?.tags_string(); + table.push(format!( + " {:?}\t{}\t{} ({} new)\t{}\t{} ({:.1}%)", + self.node_id_vec[*n], + tags_n, + stored_partitions[*n], + new_partitions[*n], + ByteSize::b(total_cap_n).to_string_as(false), + ByteSize::b(available_cap_n).to_string_as(false), + (available_cap_n as f32) / (total_cap_n as f32) * 100.0, + )); + } + + table.push(format!( + " TOTAL\t\t{} ({} unique)\t{}\t{} ({:.1}%)", + replicated_partitions, + stored_partitions_zone[z], + //new_partitions_zone[z], + ByteSize::b(total_cap_z).to_string_as(false), + ByteSize::b(available_cap_z).to_string_as(false), + percent_cap_z + )); + table.push("".into()); + } + msg.push(format_table::format_table_to_string(table)); + + Ok(msg) + } +} diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index 5aec92c0..b5b31c05 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -13,7 +13,6 @@ mod kubernetes; pub mod layout; pub mod replication_mode; -pub mod ring; pub mod system; pub mod rpc_helper; diff --git a/src/rpc/replication_mode.rs b/src/rpc/replication_mode.rs index e244e063..a3a94085 100644 --- a/src/rpc/replication_mode.rs +++ b/src/rpc/replication_mode.rs @@ -1,57 +1,94 @@ -#[derive(Clone, Copy)] -pub enum ReplicationMode { - None, - TwoWay, - TwoWayDangerous, - ThreeWay, - ThreeWayDegraded, - ThreeWayDangerous, +use garage_util::config::Config; +use garage_util::crdt::AutoCrdt; +use garage_util::error::*; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ReplicationFactor(usize); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ConsistencyMode { + /// Read- and Write-quorum are 1 + Dangerous, + /// Read-quorum is 1 + Degraded, + /// Read- and Write-quorum are determined for read-after-write-consistency + #[default] + Consistent, } -impl ReplicationMode { - pub fn parse(v: &str) -> Option { - match v { - "none" | "1" => Some(Self::None), - "2" => Some(Self::TwoWay), - "2-dangerous" => Some(Self::TwoWayDangerous), - "3" => Some(Self::ThreeWay), - "3-degraded" => Some(Self::ThreeWayDegraded), - "3-dangerous" => Some(Self::ThreeWayDangerous), - _ => None, - } +impl ConsistencyMode { + pub fn parse(s: &str) -> Option { + serde_json::from_value(serde_json::Value::String(s.to_string())).ok() } +} - pub fn control_write_max_faults(&self) -> usize { - match self { - Self::None => 0, - _ => 1, +impl AutoCrdt for ConsistencyMode { + const WARN_IF_DIFFERENT: bool = true; +} + +impl ReplicationFactor { + pub fn new(replication_factor: usize) -> Option { + if replication_factor < 1 { + None + } else { + Some(Self(replication_factor)) } } pub fn replication_factor(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay | Self::TwoWayDangerous => 2, - Self::ThreeWay | Self::ThreeWayDegraded | Self::ThreeWayDangerous => 3, + self.0 + } + + pub fn read_quorum(&self, consistency_mode: ConsistencyMode) -> usize { + match consistency_mode { + ConsistencyMode::Dangerous | ConsistencyMode::Degraded => 1, + ConsistencyMode::Consistent => self.replication_factor().div_ceil(2), } } - pub fn read_quorum(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay | Self::TwoWayDangerous => 1, - Self::ThreeWay => 2, - Self::ThreeWayDegraded | Self::ThreeWayDangerous => 1, - } - } - - pub fn write_quorum(&self) -> usize { - match self { - Self::None => 1, - Self::TwoWay => 2, - Self::TwoWayDangerous => 1, - Self::ThreeWay | Self::ThreeWayDegraded => 2, - Self::ThreeWayDangerous => 1, + pub fn write_quorum(&self, consistency_mode: ConsistencyMode) -> usize { + match consistency_mode { + ConsistencyMode::Dangerous => 1, + ConsistencyMode::Degraded | ConsistencyMode::Consistent => { + (self.replication_factor() + 1) - self.read_quorum(ConsistencyMode::Consistent) + } } } } + +impl std::convert::From for usize { + fn from(replication_factor: ReplicationFactor) -> usize { + replication_factor.0 + } +} + +pub fn parse_replication_mode( + config: &Config, +) -> Result<(ReplicationFactor, ConsistencyMode), Error> { + match (&config.replication_mode, config.replication_factor, config.consistency_mode.as_str()) { + (Some(replication_mode), None, "consistent") => { + tracing::warn!("Legacy config option replication_mode in use. Please migrate to replication_factor and consistency_mode"); + let parsed_replication_mode = match replication_mode.as_str() { + "1" | "none" => Some((ReplicationFactor(1), ConsistencyMode::Consistent)), + "2" => Some((ReplicationFactor(2), ConsistencyMode::Consistent)), + "2-dangerous" => Some((ReplicationFactor(2), ConsistencyMode::Dangerous)), + "3" => Some((ReplicationFactor(3), ConsistencyMode::Consistent)), + "3-degraded" => Some((ReplicationFactor(3), ConsistencyMode::Degraded)), + "3-dangerous" => Some((ReplicationFactor(3), ConsistencyMode::Dangerous)), + _ => None, + }; + Some(parsed_replication_mode.ok_or_message("Invalid replication_mode in config file.")?) + }, + (None, Some(replication_factor), consistency_mode) => { + let replication_factor = ReplicationFactor::new(replication_factor) + .ok_or_message("Invalid replication_factor in config file.")?; + let consistency_mode = ConsistencyMode::parse(consistency_mode) + .ok_or_message("Invalid consistency_mode in config file.")?; + Some((replication_factor, consistency_mode)) + } + _ => None, + }.ok_or_message("Either the legacy replication_mode or replication_level and consistency_mode can be set, not both.") +} diff --git a/src/rpc/ring.rs b/src/rpc/ring.rs deleted file mode 100644 index 73a126a2..00000000 --- a/src/rpc/ring.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Module containing types related to computing nodes which should receive a copy of data blocks -//! and metadata -use std::convert::TryInto; - -use garage_util::data::*; - -use crate::layout::ClusterLayout; - -/// A partition id, which is stored on 16 bits -/// i.e. we have up to 2**16 partitions. -/// (in practice we have exactly 2**PARTITION_BITS partitions) -pub type Partition = u16; - -// TODO: make this constant parametrizable in the config file -// For deployments with many nodes it might make sense to bump -// it up to 10. -// Maximum value : 16 -/// How many bits from the hash are used to make partitions. Higher numbers means more fairness in -/// presence of numerous nodes, but exponentially bigger ring. Max 16 -pub const PARTITION_BITS: usize = 8; - -const PARTITION_MASK_U16: u16 = ((1 << PARTITION_BITS) - 1) << (16 - PARTITION_BITS); - -/// A ring distributing fairly objects to nodes -#[derive(Clone)] -pub struct Ring { - /// The replication factor for this ring - pub replication_factor: usize, - - /// The network configuration used to generate this ring - pub layout: ClusterLayout, - - // Internal order of nodes used to make a more compact representation of the ring - nodes: Vec, - - // The list of entries in the ring - ring: Vec, -} - -// Type to store compactly the id of a node in the system -// Change this to u16 the day we want to have more than 256 nodes in a cluster -pub type CompactNodeType = u8; - -// The maximum number of times an object might get replicated -// This must be at least 3 because Garage supports 3-way replication -// Here we use 6 so that the size of a ring entry is 8 bytes -// (2 bytes partition id, 6 bytes node numbers as u8s) -const MAX_REPLICATION: usize = 6; - -/// An entry in the ring -#[derive(Clone, Debug)] -struct RingEntry { - // The two first bytes of the first hash that goes in this partition - // (the next bytes are zeroes) - hash_prefix: u16, - // The nodes that store this partition, stored as a list of positions in the `nodes` - // field of the Ring structure - // Only items 0 up to ring.replication_factor - 1 are used, others are zeros - nodes_buf: [CompactNodeType; MAX_REPLICATION], -} - -impl Ring { - pub(crate) fn new(layout: ClusterLayout, replication_factor: usize) -> Self { - if replication_factor != layout.replication_factor { - warn!("Could not build ring: replication factor does not match between local configuration and network role assignation."); - return Self::empty(layout, replication_factor); - } - - if layout.ring_assignation_data.len() != replication_factor * (1 << PARTITION_BITS) { - warn!("Could not build ring: network role assignation data has invalid length"); - return Self::empty(layout, replication_factor); - } - - let nodes = layout.node_id_vec.clone(); - let ring = (0..(1 << PARTITION_BITS)) - .map(|i| { - let top = (i as u16) << (16 - PARTITION_BITS); - let mut nodes_buf = [0u8; MAX_REPLICATION]; - nodes_buf[..replication_factor].copy_from_slice( - &layout.ring_assignation_data - [replication_factor * i..replication_factor * (i + 1)], - ); - RingEntry { - hash_prefix: top, - nodes_buf, - } - }) - .collect::>(); - - Self { - replication_factor, - layout, - nodes, - ring, - } - } - - fn empty(layout: ClusterLayout, replication_factor: usize) -> Self { - Self { - replication_factor, - layout, - nodes: vec![], - ring: vec![], - } - } - - /// Get the partition in which data would fall on - pub fn partition_of(&self, position: &Hash) -> Partition { - let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); - top >> (16 - PARTITION_BITS) - } - - /// Get the list of partitions and the first hash of a partition key that would fall in it - pub fn partitions(&self) -> Vec<(Partition, Hash)> { - let mut ret = vec![]; - - for (i, entry) in self.ring.iter().enumerate() { - let mut location = [0u8; 32]; - location[..2].copy_from_slice(&u16::to_be_bytes(entry.hash_prefix)[..]); - ret.push((i as u16, location.into())); - } - if !ret.is_empty() { - assert_eq!(ret[0].1, [0u8; 32].into()); - } - - ret - } - - /// Walk the ring to find the n servers in which data should be replicated - pub fn get_nodes(&self, position: &Hash, n: usize) -> Vec { - if self.ring.len() != 1 << PARTITION_BITS { - warn!("Ring not yet ready, read/writes will be lost!"); - return vec![]; - } - - let partition_idx = self.partition_of(position) as usize; - let partition = &self.ring[partition_idx]; - - let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap()); - // Check that we haven't messed up our partition table, i.e. that this partition - // table entrey indeed corresponds to the item we are storing - assert_eq!( - partition.hash_prefix & PARTITION_MASK_U16, - top & PARTITION_MASK_U16 - ); - - assert!(n <= self.replication_factor); - partition.nodes_buf[..n] - .iter() - .map(|i| self.nodes[*i as usize]) - .collect::>() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ring_entry_size() { - assert_eq!(std::mem::size_of::(), 8); - } -} diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index e59c372a..2505c2ce 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -1,12 +1,12 @@ //! Contain structs related to making RPCs -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; use std::time::Duration; use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; use futures::stream::StreamExt; use tokio::select; -use tokio::sync::watch; use opentelemetry::KeyValue; use opentelemetry::{ @@ -14,35 +14,36 @@ use opentelemetry::{ Context, }; -pub use netapp::endpoint::{Endpoint, EndpointHandler, StreamingEndpointHandler}; -pub use netapp::message::{ +pub use garage_net::endpoint::{Endpoint, EndpointHandler, StreamingEndpointHandler}; +pub use garage_net::message::{ IntoReq, Message as Rpc, OrderTag, Req, RequestPriority, Resp, PRIO_BACKGROUND, PRIO_HIGH, PRIO_NORMAL, PRIO_SECONDARY, }; -use netapp::peering::fullmesh::FullMeshPeeringStrategy; -pub use netapp::{self, NetApp, NodeID}; +use garage_net::peering::PeeringManager; +pub use garage_net::{self, NetApp, NodeID}; use garage_util::data::*; use garage_util::error::Error; use garage_util::metrics::RecordDuration; +use crate::layout::{LayoutHelper, LayoutVersion}; use crate::metrics::RpcMetrics; -use crate::ring::Ring; // Default RPC timeout = 5 minutes const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); /// Strategy to apply when making RPC -#[derive(Copy, Clone)] -pub struct RequestStrategy { +pub struct RequestStrategy { /// Min number of response to consider the request successful - pub rs_quorum: Option, - /// Should requests be dropped after enough response are received - pub rs_interrupt_after_quorum: bool, + rs_quorum: Option, + /// Send all requests at once + rs_send_all_at_once: Option, /// Request priority - pub rs_priority: RequestPriority, + rs_priority: RequestPriority, /// Custom timeout for this request rs_timeout: Timeout, + /// Data to drop when everything completes + rs_drop_on_complete: T, } #[derive(Copy, Clone)] @@ -52,25 +53,50 @@ enum Timeout { Custom(Duration), } -impl RequestStrategy { +impl Clone for RequestStrategy<()> { + fn clone(&self) -> Self { + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: (), + } + } +} + +impl RequestStrategy<()> { /// Create a RequestStrategy with default timeout and not interrupting when quorum reached pub fn with_priority(prio: RequestPriority) -> Self { RequestStrategy { rs_quorum: None, - rs_interrupt_after_quorum: false, + rs_send_all_at_once: None, rs_priority: prio, rs_timeout: Timeout::Default, + rs_drop_on_complete: (), } } + /// Add an item to be dropped on completion + pub fn with_drop_on_completion(self, drop_on_complete: T) -> RequestStrategy { + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: drop_on_complete, + } + } +} + +impl RequestStrategy { /// Set quorum to be reached for request pub fn with_quorum(mut self, quorum: usize) -> Self { self.rs_quorum = Some(quorum); self } - /// Set if requests can be dropped after quorum has been reached - /// In general true for read requests, and false for write - pub fn interrupt_after_quorum(mut self, interrupt: bool) -> Self { - self.rs_interrupt_after_quorum = interrupt; + /// Set quorum to be reached for request + pub fn send_all_at_once(mut self, value: bool) -> Self { + self.rs_send_all_at_once = Some(value); self } /// Deactivate timeout for this request @@ -83,6 +109,19 @@ impl RequestStrategy { self.rs_timeout = Timeout::Custom(timeout); self } + /// Extract drop_on_complete item + fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) { + ( + RequestStrategy { + rs_quorum: self.rs_quorum, + rs_send_all_at_once: self.rs_send_all_at_once, + rs_priority: self.rs_priority, + rs_timeout: self.rs_timeout, + rs_drop_on_complete: (), + }, + self.rs_drop_on_complete, + ) + } } #[derive(Clone)] @@ -90,8 +129,8 @@ pub struct RpcHelper(Arc); struct RpcHelperInner { our_node_id: Uuid, - fullmesh: Arc, - ring: watch::Receiver>, + peering: Arc, + layout: Arc>, metrics: RpcMetrics, rpc_timeout: Duration, } @@ -99,16 +138,16 @@ struct RpcHelperInner { impl RpcHelper { pub(crate) fn new( our_node_id: Uuid, - fullmesh: Arc, - ring: watch::Receiver>, + peering: Arc, + layout: Arc>, rpc_timeout: Option, ) -> Self { let metrics = RpcMetrics::new(); Self(Arc::new(RpcHelperInner { our_node_id, - fullmesh, - ring, + peering, + layout, metrics, rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT), })) @@ -123,13 +162,19 @@ impl RpcHelper { endpoint: &Endpoint, to: Uuid, msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result where M: Rpc>, N: IntoReq + Send, H: StreamingEndpointHandler, { + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!("RPC [{}] to {:?}", endpoint.path(), to); + let mut span = tracer.start(span_name); + span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); + span.set_attribute(KeyValue::new("to", format!("{:?}", to))); + let metric_tags = [ KeyValue::new("rpc_endpoint", endpoint.path().to_string()), KeyValue::new("from", format!("{:?}", self.0.our_node_id)), @@ -141,6 +186,7 @@ impl RpcHelper { let node_id = to.into(); let rpc_call = endpoint .call_streaming(&node_id, msg, strat.rs_priority) + .with_context(Context::current_with_span(span)) .record_duration(&self.0.metrics.rpc_duration, &metric_tags); let timeout = async { @@ -176,19 +222,24 @@ impl RpcHelper { endpoint: &Endpoint, to: &[Uuid], msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result)>, Error> where M: Rpc>, N: IntoReq, H: StreamingEndpointHandler, { - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!("RPC [{}] call_many {} nodes", endpoint.path(), to.len()); + let span = tracer.start(span_name); + + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; let resps = join_all( to.iter() - .map(|to| self.call(endpoint, *to, msg.clone(), strat)), + .map(|to| self.call(endpoint, *to, msg.clone(), strat.clone())), ) + .with_context(Context::current_with_span(span)) .await; Ok(to .iter() @@ -201,7 +252,7 @@ impl RpcHelper { &self, endpoint: &Endpoint, msg: N, - strat: RequestStrategy, + strat: RequestStrategy<()>, ) -> Result)>, Error> where M: Rpc>, @@ -210,7 +261,7 @@ impl RpcHelper { { let to = self .0 - .fullmesh + .peering .get_peer_list() .iter() .map(|p| p.id.into()) @@ -220,12 +271,28 @@ impl RpcHelper { /// Make a RPC call to multiple servers, returning either a Vec of responses, /// or an error if quorum could not be reached due to too many errors + /// + /// If RequestStrategy has send_all_at_once set, then all requests will be + /// sent at once, and `try_call_many` will return as soon as a quorum of + /// responses is achieved, dropping and cancelling the remaining requests. + /// + /// Otherwise, `quorum` requests will be sent at the same time, and if an + /// error response is received, a new request will be sent to replace it. + /// The ordering of nodes to which requests are sent is determined by + /// the `RpcHelper::request_order` function, which takes into account + /// parameters such as node zones and measured ping values. + /// + /// In both cases, the basic contract of this function is that even in the + /// absence of failures, the RPC call might not be driven to completion + /// on all of the specified nodes. It is therefore unfit for broadcast + /// write operations where we expect all nodes to successfully store + /// the written date. pub async fn try_call_many( &self, endpoint: &Arc>, to: &[Uuid], msg: N, - strategy: RequestStrategy, + strategy: RequestStrategy<()>, ) -> Result, Error> where M: Rpc> + 'static, @@ -236,36 +303,29 @@ impl RpcHelper { let quorum = strategy.rs_quorum.unwrap_or(to.len()); let tracer = opentelemetry::global::tracer("garage"); - let span_name = if strategy.rs_interrupt_after_quorum { - format!("RPC {} to {} of {}", endpoint.path(), quorum, to.len()) - } else { - format!( - "RPC {} to {} (quorum {})", - endpoint.path(), - to.len(), - quorum - ) - }; + let span_name = format!( + "RPC [{}] try_call_many (quorum {}/{})", + endpoint.path(), + quorum, + to.len() + ); + let mut span = tracer.start(span_name); span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); - span.set_attribute(KeyValue::new( - "interrupt_after_quorum", - strategy.rs_interrupt_after_quorum.to_string(), - )); - self.try_call_many_internal(endpoint, to, msg, strategy, quorum) + self.try_call_many_inner(endpoint, to, msg, strategy, quorum) .with_context(Context::current_with_span(span)) .await } - async fn try_call_many_internal( + async fn try_call_many_inner( &self, endpoint: &Arc>, to: &[Uuid], msg: N, - strategy: RequestStrategy, + strategy: RequestStrategy<()>, quorum: usize, ) -> Result, Error> where @@ -274,110 +334,65 @@ impl RpcHelper { H: StreamingEndpointHandler + 'static, S: Send + 'static, { - let msg = msg.into_req().map_err(netapp::error::Error::from)?; + // Once quorum is reached, other requests don't matter. + // What we do here is only send the required number of requests + // to reach a quorum, priorizing nodes with the lowest latency. + // When there are errors, we start new requests to compensate. + + // TODO: this could be made more aggressive, e.g. if after 2x the + // average ping of a given request, the response is not yet received, + // preemptively send an additional request to any remaining nodes. + + // Reorder requests to priorize closeness / low latency + let request_order = + self.request_order(&self.0.layout.read().unwrap().current(), to.iter().copied()); + let send_all_at_once = strategy.rs_send_all_at_once.unwrap_or(false); // Build future for each request // They are not started now: they are added below in a FuturesUnordered // object that will take care of polling them (see below) - let requests = to.iter().cloned().map(|to| { + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; + let mut requests = request_order.into_iter().map(|to| { let self2 = self.clone(); let msg = msg.clone(); let endpoint2 = endpoint.clone(); - (to, async move { - self2.call(&endpoint2, to, msg, strategy).await - }) + let strategy = strategy.clone(); + async move { self2.call(&endpoint2, to, msg, strategy).await } }); // Vectors in which success results and errors will be collected let mut successes = vec![]; let mut errors = vec![]; - if strategy.rs_interrupt_after_quorum { - // Case 1: once quorum is reached, other requests don't matter. - // What we do here is only send the required number of requests - // to reach a quorum, priorizing nodes with the lowest latency. - // When there are errors, we start new requests to compensate. + // resp_stream will contain all of the requests that are currently in flight. + // (for the moment none, they will be added in the loop below) + let mut resp_stream = FuturesUnordered::new(); - // Reorder requests to priorize closeness / low latency - let request_order = self.request_order(to); - let mut ord_requests = vec![(); request_order.len()] - .into_iter() - .map(|_| None) - .collect::>(); - for (to, fut) in requests { - let i = request_order.iter().position(|x| *x == to).unwrap(); - ord_requests[i] = Some((to, fut)); - } - - // Make an iterator to take requests in their sorted order - let mut requests = ord_requests.into_iter().map(Option::unwrap); - - // resp_stream will contain all of the requests that are currently in flight. - // (for the moment none, they will be added in the loop below) - let mut resp_stream = FuturesUnordered::new(); - - // Do some requests and collect results - 'request_loop: while successes.len() < quorum { - // If the current set of requests that are running is not enough to possibly - // reach quorum, start some new requests. - while successes.len() + resp_stream.len() < quorum { - if let Some((req_to, fut)) = requests.next() { - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer.start(format!("RPC to {:?}", req_to)); - resp_stream.push(tokio::spawn( - fut.with_context(Context::current_with_span(span)), - )); - } else { - // If we have no request to add, we know that we won't ever - // reach quorum: bail out now. - break 'request_loop; - } - } - assert!(!resp_stream.is_empty()); // because of loop invariants - - // Wait for one request to terminate - match resp_stream.next().await.unwrap().unwrap() { - Ok(msg) => { - successes.push(msg); - } - Err(e) => { - errors.push(e); - } - } - } - } else { - // Case 2: all of the requests need to be sent in all cases, - // and need to terminate. (this is the case for writes that - // must be spread to n nodes) - // Just start all the requests in parallel and return as soon - // as the quorum is reached. - let mut resp_stream = requests - .map(|(_, fut)| fut) - .collect::>(); - - while let Some(resp) = resp_stream.next().await { - match resp { - Ok(msg) => { - successes.push(msg); - if successes.len() >= quorum { - break; - } - } - Err(e) => { - errors.push(e); - } + // Do some requests and collect results + while successes.len() < quorum { + // If the current set of requests that are running is not enough to possibly + // reach quorum, start some new requests. + while send_all_at_once || successes.len() + resp_stream.len() < quorum { + if let Some(fut) = requests.next() { + resp_stream.push(fut) + } else { + break; } } - if !resp_stream.is_empty() { - // Continue remaining requests in background. - // Note: these requests can get interrupted on process shutdown, - // we must not count on them being executed for certain. - // For all background things that have to happen with certainty, - // they have to be put in a proper queue that is persisted to disk. - tokio::spawn(async move { - resp_stream.collect::>>().await; - }); + if successes.len() + resp_stream.len() < quorum { + // We know we won't ever reach quorum + break; + } + + // Wait for one request to terminate + match resp_stream.next().await.unwrap() { + Ok(msg) => { + successes.push(msg); + } + Err(e) => { + errors.push(e); + } } } @@ -385,19 +400,233 @@ impl RpcHelper { Ok(successes) } else { let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) + Err(Error::Quorum( + quorum, + None, + successes.len(), + to.len(), + errors, + )) } } - pub fn request_order(&self, nodes: &[Uuid]) -> Vec { - // Retrieve some status variables that we will use to sort requests - let peer_list = self.0.fullmesh.get_peer_list(); - let ring: Arc = self.0.ring.borrow().clone(); - let our_zone = match ring.layout.node_role(&self.0.our_node_id) { - Some(pc) => &pc.zone, - None => "", + /// Make a RPC call to multiple servers, returning either a Vec of responses, + /// or an error if quorum could not be reached due to too many errors + /// + /// Contrary to try_call_many, this function is especially made for broadcast + /// write operations. In particular: + /// + /// - The request are sent to all specified nodes as soon as `try_write_many_sets` + /// is invoked. + /// + /// - When `try_write_many_sets` returns, all remaining requests that haven't + /// completed move to a background task so that they have a chance to + /// complete successfully if there are no failures. + /// + /// In addition, the nodes to which requests should be sent are divided in + /// "quorum sets", and `try_write_many_sets` only returns once a quorum + /// has been validated in each set. This is used in the case of cluster layout + /// changes, where data has to be written both in the old layout and in the + /// new one as long as all nodes have not successfully tranisitionned and + /// moved all data to the new layout. + pub async fn try_write_many_sets( + &self, + endpoint: &Arc>, + to_sets: &[Vec], + msg: N, + strategy: RequestStrategy, + ) -> Result, Error> + where + M: Rpc> + 'static, + N: IntoReq, + H: StreamingEndpointHandler + 'static, + S: Send + 'static, + T: Send + 'static, + { + let quorum = strategy + .rs_quorum + .expect("internal error: missing quorum value in try_write_many_sets"); + + let tracer = opentelemetry::global::tracer("garage"); + let span_name = format!( + "RPC [{}] try_write_many_sets (quorum {} in {} sets)", + endpoint.path(), + quorum, + to_sets.len() + ); + + let mut span = tracer.start(span_name); + span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); + span.set_attribute(KeyValue::new("to", format!("{:?}", to_sets))); + span.set_attribute(KeyValue::new("quorum", quorum as i64)); + + self.try_write_many_sets_inner(endpoint, to_sets, msg, strategy, quorum) + .with_context(Context::current_with_span(span)) + .await + } + + async fn try_write_many_sets_inner( + &self, + endpoint: &Arc>, + to_sets: &[Vec], + msg: N, + strategy: RequestStrategy, + quorum: usize, + ) -> Result, Error> + where + M: Rpc> + 'static, + N: IntoReq, + H: StreamingEndpointHandler + 'static, + S: Send + 'static, + T: Send + 'static, + { + // Peers may appear in many quorum sets. Here, build a list of peers, + // mapping to the index of the quorum sets in which they appear. + let mut result_tracker = QuorumSetResultTracker::new(to_sets, quorum); + + let (strategy, drop_on_complete) = strategy.extract_drop_on_complete(); + + // Send one request to each peer of the quorum sets + let msg = msg.into_req().map_err(garage_net::error::Error::from)?; + let requests = result_tracker.nodes.keys().map(|peer| { + let self2 = self.clone(); + let msg = msg.clone(); + let endpoint2 = endpoint.clone(); + let to = *peer; + let strategy = strategy.clone(); + async move { (to, self2.call(&endpoint2, to, msg, strategy).await) } + }); + let mut resp_stream = requests.collect::>(); + + // Drive requests to completion + while let Some((node, resp)) = resp_stream.next().await { + // Store the response in the correct vector and increment the + // appropriate counters + result_tracker.register_result(node, resp); + + // If we have a quorum of ok in all quorum sets, then it's a success! + if result_tracker.all_quorums_ok() { + // Continue all other requests in background + tokio::spawn(async move { + resp_stream.collect::)>>().await; + drop(drop_on_complete); + }); + + return Ok(result_tracker.success_values()); + } + + // If there is a quorum set for which too many errors were received, + // we know it's impossible to get a quorum, so return immediately. + if result_tracker.too_many_failures() { + break; + } + } + + // At this point, there is no quorum and we know that a quorum + // will never be achieved. Currently, we drop all remaining requests. + // Should we still move them to background so that they can continue + // for non-failed nodes? Not doing so has no impact on correctness, + // but it means that more cancellation messages will be sent. Idk. + // (When an in-progress request future is dropped, Netapp automatically + // sends a cancellation message to the remote node to inform it that + // the result is no longer needed. In turn, if the remote node receives + // the cancellation message in time, it interrupts the task of the + // running request handler.) + + // Failure, could not get quorum + Err(result_tracker.quorum_error()) + } + + // ---- functions not related to MAKING RPCs, but just determining to what nodes + // they should be made and in which order ---- + + /// Determine to what nodes, and in what order, requests to read a data block + /// should be sent. All nodes in the Vec returned by this function are tried + /// one by one until there is one that returns the block (in block/manager.rs). + /// + /// We want to have the best chance of finding the block in as few requests + /// as possible, and we want to avoid nodes that answer slowly. + /// + /// Note that when there are several active layout versions, the block might + /// be stored only by nodes of the latest version (in case of a block that was + /// written after the layout change), or only by nodes of the oldest active + /// version (for all blocks that were written before). So we have to try nodes + /// of all layout versions. We also want to try nodes of all layout versions + /// fast, so as to optimize the chance of finding the block fast. + /// + /// Therefore, the strategy is the following: + /// + /// 1. ask first all nodes of all currently active layout versions + /// -> ask the preferred node in all layout versions (older to newer), + /// then the second preferred onde in all verions, etc. + /// -> we start by the oldest active layout version first, because a majority + /// of blocks will have been saved before the layout change + /// 2. ask all nodes of historical layout versions, for blocks which have not + /// yet been transferred to their new storage nodes + /// + /// The preference order, for each layout version, is given by `request_order`, + /// based on factors such as nodes being in the same datacenter, + /// having low ping, etc. + pub fn block_read_nodes_of(&self, position: &Hash, rpc_helper: &RpcHelper) -> Vec { + let layout = self.0.layout.read().unwrap(); + + // Compute, for each layout version, the set of nodes that might store + // the block, and put them in their preferred order as of `request_order`. + let mut vernodes = layout.versions().iter().map(|ver| { + let nodes = ver.nodes_of(position, ver.replication_factor); + rpc_helper.request_order(layout.current(), nodes) + }); + + let mut ret = if layout.versions().len() == 1 { + // If we have only one active layout version, then these are the + // only nodes we ask in step 1 + vernodes.next().unwrap() + } else { + let vernodes = vernodes.collect::>(); + + let mut nodes = Vec::::with_capacity(12); + for i in 0..layout.current().replication_factor { + for vn in vernodes.iter() { + if let Some(n) = vn.get(i) { + if !nodes.contains(&n) { + if *n == self.0.our_node_id { + // it's always fast (almost free) to ask locally, + // so always put that as first choice + nodes.insert(0, *n); + } else { + nodes.push(*n); + } + } + } + } + } + + nodes }; + // Second step: add nodes of older layout versions + let old_ver_iter = layout.inner().old_versions.iter().rev(); + for ver in old_ver_iter { + let nodes = ver.nodes_of(position, ver.replication_factor); + for node in rpc_helper.request_order(layout.current(), nodes) { + if !ret.contains(&node) { + ret.push(node); + } + } + } + + ret + } + + fn request_order( + &self, + layout: &LayoutVersion, + nodes: impl Iterator, + ) -> Vec { + // Retrieve some status variables that we will use to sort requests + let peer_list = self.0.peering.get_peer_list(); + let our_zone = layout.get_node_zone(&self.0.our_node_id).unwrap_or(""); + // Augment requests with some information used to sort them. // The tuples are as follows: // (is another node?, is another zone?, latency, node ID, request future) @@ -405,22 +634,18 @@ impl RpcHelper { // By sorting this vec, we priorize ourself, then nodes in the same zone, // and within a same zone we priorize nodes with the lowest latency. let mut nodes = nodes - .iter() .map(|to| { - let peer_zone = match ring.layout.node_role(to) { - Some(pc) => &pc.zone, - None => "", - }; + let peer_zone = layout.get_node_zone(&to).unwrap_or(""); let peer_avg_ping = peer_list .iter() .find(|x| x.id.as_ref() == to.as_slice()) .and_then(|pi| pi.avg_ping) .unwrap_or_else(|| Duration::from_secs(10)); ( - *to != self.0.our_node_id, + to != self.0.our_node_id, peer_zone != our_zone, peer_avg_ping, - *to, + to, ) }) .collect::>(); @@ -434,3 +659,108 @@ impl RpcHelper { .collect::>() } } + +// ------- utility for tracking successes/errors among write sets -------- + +pub struct QuorumSetResultTracker { + /// The set of nodes and the index of the quorum sets they belong to + pub nodes: HashMap>, + /// The quorum value, i.e. number of success responses to await in each set + pub quorum: usize, + + /// The success responses received + pub successes: Vec<(Uuid, S)>, + /// The error responses received + pub failures: Vec<(Uuid, E)>, + + /// The counters for successes in each set + pub success_counters: Box<[usize]>, + /// The counters for failures in each set + pub failure_counters: Box<[usize]>, + /// The total number of nodes in each set + pub set_lens: Box<[usize]>, +} + +impl QuorumSetResultTracker +where + E: std::fmt::Display, +{ + pub fn new(sets: &[A], quorum: usize) -> Self + where + A: AsRef<[Uuid]>, + { + let mut nodes = HashMap::>::new(); + for (i, set) in sets.iter().enumerate() { + for node in set.as_ref().iter() { + nodes.entry(*node).or_default().push(i); + } + } + + let num_nodes = nodes.len(); + Self { + nodes, + quorum, + successes: Vec::with_capacity(num_nodes), + failures: vec![], + success_counters: vec![0; sets.len()].into_boxed_slice(), + failure_counters: vec![0; sets.len()].into_boxed_slice(), + set_lens: sets + .iter() + .map(|x| x.as_ref().len()) + .collect::>() + .into_boxed_slice(), + } + } + + pub fn register_result(&mut self, node: Uuid, result: Result) { + match result { + Ok(s) => { + self.successes.push((node, s)); + for set in self.nodes.get(&node).unwrap().iter() { + self.success_counters[*set] += 1; + } + } + Err(e) => { + self.failures.push((node, e)); + for set in self.nodes.get(&node).unwrap().iter() { + self.failure_counters[*set] += 1; + } + } + } + } + + pub fn all_quorums_ok(&self) -> bool { + self.success_counters + .iter() + .all(|ok_cnt| *ok_cnt >= self.quorum) + } + + pub fn too_many_failures(&self) -> bool { + self.failure_counters + .iter() + .zip(self.set_lens.iter()) + .any(|(err_cnt, set_len)| *err_cnt + self.quorum > *set_len) + } + + pub fn success_values(self) -> Vec { + self.successes + .into_iter() + .map(|(_, x)| x) + .collect::>() + } + + pub fn quorum_error(self) -> Error { + let errors = self + .failures + .iter() + .map(|(n, e)| format!("{:?}: {}", n, e)) + .collect::>(); + Error::Quorum( + self.quorum, + Some(self.set_lens.len()), + self.successes.len(), + self.nodes.len(), + errors, + ) + } +} diff --git a/src/rpc/system.rs b/src/rpc/system.rs index b42e49fc..2a52ae5d 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -1,30 +1,27 @@ //! Module containing structs related to membership management -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::{Read, Write}; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; -use std::sync::atomic::Ordering; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, RwLockReadGuard}; use std::time::{Duration, Instant}; -use arc_swap::ArcSwap; -use async_trait::async_trait; -use futures::{join, select}; -use futures_util::future::*; +use arc_swap::ArcSwapOption; +use futures::join; use serde::{Deserialize, Serialize}; use sodiumoxide::crypto::sign::ed25519; -use tokio::sync::watch; -use tokio::sync::Mutex; +use tokio::select; +use tokio::sync::{watch, Notify}; -use netapp::endpoint::{Endpoint, EndpointHandler}; -use netapp::message::*; -use netapp::peering::fullmesh::FullMeshPeeringStrategy; -use netapp::util::parse_and_resolve_peer_addr_async; -use netapp::{NetApp, NetworkKey, NodeID, NodeKey}; +use garage_net::endpoint::{Endpoint, EndpointHandler}; +use garage_net::message::*; +use garage_net::peering::{PeerConnState, PeeringManager}; +use garage_net::util::parse_and_resolve_peer_addr_async; +use garage_net::{NetApp, NetworkKey, NodeID, NodeKey}; -use garage_util::config::Config; #[cfg(feature = "kubernetes-discovery")] use garage_util::config::KubernetesDiscoveryConfig; +use garage_util::config::{Config, DataDirEnum}; use garage_util::data::*; use garage_util::error::*; use garage_util::persister::Persister; @@ -34,9 +31,10 @@ use garage_util::time::*; use crate::consul::ConsulDiscovery; #[cfg(feature = "kubernetes-discovery")] use crate::kubernetes::*; -use crate::layout::*; +use crate::layout::{ + self, manager::LayoutManager, LayoutHelper, LayoutHistory, NodeRoleV, RpcLayoutDigest, +}; use crate::replication_mode::*; -use crate::ring::*; use crate::rpc_helper::*; use crate::system_metrics::*; @@ -47,29 +45,34 @@ const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); /// Version tag used for version check upon Netapp connection. /// Cluster nodes with different version tags are deemed /// incompatible and will refuse to connect. -pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008 +pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650010; // garage 0x0010 (1.0) /// RPC endpoint used for calls related to membership -pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; +pub const SYSTEM_RPC_PATH: &str = "garage_rpc/system.rs/SystemRpc"; /// RPC messages related to membership #[derive(Debug, Serialize, Deserialize, Clone)] pub enum SystemRpc { - /// Response to successfull advertisements + /// Response to successful advertisements Ok, - /// Request to connect to a specific node (in @: format) + /// Request to connect to a specific node (in @: format, pubkey = full-length node ID) Connect(String), - /// Ask other node its cluster layout. Answered with AdvertiseClusterLayout - PullClusterLayout, /// Advertise Garage status. Answered with another AdvertiseStatus. /// Exchanged with every node on a regular basis. AdvertiseStatus(NodeStatus), - /// Advertisement of cluster layout. Sent spontanously or in response to PullClusterLayout - AdvertiseClusterLayout(ClusterLayout), /// Get known nodes states GetKnownNodes, /// Return known nodes ReturnKnownNodes(Vec), + + /// Ask other node its cluster layout. Answered with AdvertiseClusterLayout + PullClusterLayout, + /// Advertisement of cluster layout. Sent spontanously or in response to PullClusterLayout + AdvertiseClusterLayout(LayoutHistory), + /// Ask other node its cluster layout update trackers. + PullClusterLayoutTrackers, + /// Advertisement of cluster layout update trackers. + AdvertiseClusterLayoutTrackers(layout::UpdateTrackers), } impl Rpc for SystemRpc { @@ -85,20 +88,17 @@ pub struct System { /// The id of this node pub id: Uuid, - persist_cluster_layout: Persister, persist_peer_list: Persister, - local_status: ArcSwap, + pub(crate) local_status: RwLock, node_status: RwLock>, pub netapp: Arc, - fullmesh: Arc, - pub rpc: RpcHelper, + peering: Arc, - system_endpoint: Arc>, + pub(crate) system_endpoint: Arc>, rpc_listen_addr: SocketAddr, - #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))] rpc_public_addr: Option, bootstrap_peers: Vec, @@ -107,32 +107,28 @@ pub struct System { #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: Option, - metrics: SystemMetrics, + pub layout_manager: Arc, - replication_mode: ReplicationMode, - replication_factor: usize, + metrics: ArcSwapOption, - /// The ring - pub ring: watch::Receiver>, - update_ring: Mutex>>, + pub(crate) replication_factor: ReplicationFactor, /// Path to metadata directory pub metadata_dir: PathBuf, /// Path to data directory - pub data_dir: PathBuf, + pub data_dir: DataDirEnum, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NodeStatus { /// Hostname of the node - pub hostname: String, + pub hostname: Option, /// Replication factor configured on the node pub replication_factor: usize, - /// Cluster layout version - pub cluster_layout_version: u64, - /// Hash of cluster layout staging data - pub cluster_layout_staging_hash: Hash, + + /// Cluster layout digest + pub layout_digest: RpcLayoutDigest, /// Disk usage on partition containing metadata directory (tuple: `(avail, total)`) #[serde(default)] @@ -145,13 +141,13 @@ pub struct NodeStatus { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct KnownNodeInfo { pub id: Uuid, - pub addr: SocketAddr, + pub addr: Option, pub is_up: bool, pub last_seen_secs_ago: Option, pub status: NodeStatus, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy)] pub struct ClusterHealth { /// The current health status of the cluster (see below) pub status: ClusterHealthStatus, @@ -171,11 +167,11 @@ pub struct ClusterHealth { pub partitions_all_ok: usize, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum ClusterHealthStatus { /// All nodes are available Healthy, - /// Some storage nodes are unavailable, but quorum is stil + /// Some storage nodes are unavailable, but quorum is still /// achieved for all partitions Degraded, /// Quorum is not available for some partitions @@ -245,11 +241,11 @@ impl System { /// Create this node's membership manager pub fn new( network_key: NetworkKey, - replication_mode: ReplicationMode, + replication_factor: ReplicationFactor, + consistency_mode: ConsistencyMode, config: &Config, ) -> Result, Error> { - let replication_factor = replication_mode.replication_factor(); - + // ---- setup netapp RPC protocol ---- let node_key = gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID"); info!( @@ -257,82 +253,39 @@ impl System { hex::encode(&node_key.public_key()[..8]) ); - let persist_cluster_layout: Persister = - Persister::new(&config.metadata_dir, "cluster_layout"); - let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); + let bind_outgoing_to = Some(config) + .filter(|x| x.rpc_bind_outgoing) + .map(|x| x.rpc_bind_addr.ip()); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key, bind_outgoing_to); + let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into()); - let cluster_layout = match persist_cluster_layout.load() { - Ok(x) => { - if x.replication_factor != replication_factor { - return Err(Error::Message(format!( - "Prevous cluster layout has replication factor {}, which is different than the one specified in the config file ({}). The previous cluster layout can be purged, if you know what you are doing, simply by deleting the `cluster_layout` file in your metadata directory.", - x.replication_factor, - replication_factor - ))); - } - x - } - Err(e) => { - info!( - "No valid previous cluster layout stored ({}), starting fresh.", - e - ); - ClusterLayout::new(replication_factor) - } - }; - - let metrics = SystemMetrics::new(replication_factor); - - let mut local_status = NodeStatus::initial(replication_factor, &cluster_layout); - local_status.update_disk_usage(&config.metadata_dir, &config.data_dir, &metrics); - - let ring = Ring::new(cluster_layout, replication_factor); - let (update_ring, ring) = watch::channel(Arc::new(ring)); - - let rpc_public_addr = match &config.rpc_public_addr { - Some(a_str) => { - use std::net::ToSocketAddrs; - match a_str.to_socket_addrs() { - Err(e) => { - error!( - "Cannot resolve rpc_public_addr {} from config file: {}.", - a_str, e - ); - None - } - Ok(a) => { - let a = a.collect::>(); - if a.is_empty() { - error!("rpc_public_addr {} resolve to no known IP address", a_str); - } - if a.len() > 1 { - warn!("Multiple possible resolutions for rpc_public_addr: {:?}. Taking the first one.", a); - } - a.into_iter().next() - } - } - } - None => { - let addr = - get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); - if let Some(a) = addr { - warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); - } - addr - } - }; + // ---- setup netapp public listener and full mesh peering strategy ---- + let rpc_public_addr = get_rpc_public_addr(config); if rpc_public_addr.is_none() { warn!("This Garage node does not know its publicly reachable RPC address, this might hamper intra-cluster communication."); } - let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key); - let fullmesh = FullMeshPeeringStrategy::new(netapp.clone(), vec![], rpc_public_addr); + let peering = PeeringManager::new(netapp.clone(), vec![], rpc_public_addr); if let Some(ping_timeout) = config.rpc_ping_timeout_msec { - fullmesh.set_ping_timeout_millis(ping_timeout); + peering.set_ping_timeout_millis(ping_timeout); } - let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into()); + let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); + // ---- setup cluster layout and layout manager ---- + let layout_manager = LayoutManager::new( + config, + netapp.id, + system_endpoint.clone(), + peering.clone(), + replication_factor, + consistency_mode, + )?; + + let mut local_status = NodeStatus::initial(replication_factor, &layout_manager); + local_status.update_disk_usage(&config.metadata_dir, &config.data_dir); + + // ---- if enabled, set up additional peer discovery methods ---- #[cfg(feature = "consul-discovery")] let consul_discovery = match &config.consul_discovery { Some(cfg) => Some( @@ -351,66 +304,87 @@ impl System { warn!("Kubernetes discovery is not enabled in this build."); } + // ---- almost done ---- let sys = Arc::new(System { id: netapp.id.into(), - persist_cluster_layout, persist_peer_list, - local_status: ArcSwap::new(Arc::new(local_status)), + local_status: RwLock::new(local_status), node_status: RwLock::new(HashMap::new()), netapp: netapp.clone(), - fullmesh: fullmesh.clone(), - rpc: RpcHelper::new( - netapp.id.into(), - fullmesh, - ring.clone(), - config.rpc_timeout_msec.map(Duration::from_millis), - ), + peering: peering.clone(), system_endpoint, - replication_mode, replication_factor, rpc_listen_addr: config.rpc_bind_addr, - #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))] rpc_public_addr, bootstrap_peers: config.bootstrap_peers.clone(), #[cfg(feature = "consul-discovery")] consul_discovery, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: config.kubernetes_discovery.clone(), - metrics, + layout_manager, + metrics: ArcSwapOption::new(None), - ring, - update_ring: Mutex::new(update_ring), metadata_dir: config.metadata_dir.clone(), data_dir: config.data_dir.clone(), }); + sys.system_endpoint.set_handler(sys.clone()); + + let metrics = SystemMetrics::new(sys.clone()); + sys.metrics.store(Some(Arc::new(metrics))); + Ok(sys) } - /// Perform bootstraping, starting the ping loop + /// Perform bootstrapping, starting the ping loop pub async fn run(self: Arc, must_exit: watch::Receiver) { join!( - self.netapp - .clone() - .listen(self.rpc_listen_addr, None, must_exit.clone()), - self.fullmesh.clone().run(must_exit.clone()), + self.netapp.clone().listen( + self.rpc_listen_addr, + self.rpc_public_addr, + must_exit.clone() + ), + self.peering.clone().run(must_exit.clone()), self.discovery_loop(must_exit.clone()), self.status_exchange_loop(must_exit.clone()), ); } + pub fn cleanup(&self) { + // Break reference cycle + self.metrics.store(None); + } + + // ---- Public utilities / accessors ---- + + pub fn cluster_layout(&self) -> RwLockReadGuard<'_, LayoutHelper> { + self.layout_manager.layout() + } + + pub fn layout_notify(&self) -> Arc { + self.layout_manager.change_notify.clone() + } + + pub fn rpc_helper(&self) -> &RpcHelper { + &self.layout_manager.rpc_helper + } + // ---- Administrative operations (directly available and // also available through RPC) ---- pub fn get_known_nodes(&self) -> Vec { let node_status = self.node_status.read().unwrap(); let known_nodes = self - .fullmesh + .peering .get_peer_list() .iter() .map(|n| KnownNodeInfo { id: n.id.into(), - addr: n.addr, + addr: match n.state { + PeerConnState::Ourself => self.rpc_public_addr, + PeerConnState::Connected { addr } => Some(addr), + _ => None, + }, is_up: n.is_up(), last_seen_secs_ago: n .last_seen @@ -425,18 +399,6 @@ impl System { known_nodes } - pub fn get_cluster_layout(&self) -> ClusterLayout { - self.ring.borrow().layout.clone() - } - - pub async fn update_cluster_layout( - self: &Arc, - layout: &ClusterLayout, - ) -> Result<(), Error> { - self.handle_advertise_cluster_layout(layout).await?; - Ok(()) - } - pub async fn connect(&self, node: &str) -> Result<(), Error> { let (pubkey, addrs) = parse_and_resolve_peer_addr_async(node) .await @@ -466,47 +428,65 @@ impl System { } pub fn health(&self) -> ClusterHealth { - let ring: Arc<_> = self.ring.borrow().clone(); - let quorum = self.replication_mode.write_quorum(); - let replication_factor = self.replication_factor; + let quorum = self + .replication_factor + .write_quorum(ConsistencyMode::Consistent); + // Gather information about running nodes. + // Technically, `nodes` contains currently running nodes, as well + // as nodes that this Garage process has been connected to at least + // once since it started. let nodes = self .get_known_nodes() .into_iter() .map(|n| (n.id, n)) .collect::>(); let connected_nodes = nodes.iter().filter(|(_, n)| n.is_up).count(); + let node_up = |x: &Uuid| nodes.get(x).map(|n| n.is_up).unwrap_or(false); - let storage_nodes = ring - .layout - .roles - .items() - .iter() - .filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some())) - .collect::>(); - let storage_nodes_ok = storage_nodes - .iter() - .filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false)) - .count(); + // Acquire a rwlock read-lock to the current cluster layout + let layout = self.cluster_layout(); - let partitions = ring.partitions(); - let partitions_n_up = partitions - .iter() - .map(|(_, h)| { - let pn = ring.get_nodes(h, ring.replication_factor); - pn.iter() - .filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false)) - .count() - }) - .collect::>(); - let partitions_all_ok = partitions_n_up - .iter() - .filter(|c| **c == replication_factor) - .count(); - let partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count(); + // Obtain information about nodes that have a role as storage nodes + // in one of the active layout versions + let mut storage_nodes = HashSet::::with_capacity(16); + for ver in layout.versions().iter() { + storage_nodes.extend( + ver.roles + .items() + .iter() + .filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some())) + .map(|(n, _, _)| *n), + ) + } + let storage_nodes_ok = storage_nodes.iter().filter(|x| node_up(x)).count(); + // Determine the number of partitions that have: + // - a quorum of up nodes for all write sets (i.e. are available) + // - for which all nodes in all write sets are up (i.e. are fully healthy) + let partitions = layout.current().partitions().collect::>(); + let mut partitions_quorum = 0; + let mut partitions_all_ok = 0; + for (_, hash) in partitions.iter() { + let mut write_sets = layout + .versions() + .iter() + .map(|x| x.nodes_of(hash, x.replication_factor)); + let has_quorum = write_sets + .clone() + .all(|set| set.filter(|x| node_up(x)).count() >= quorum); + let all_ok = write_sets.all(|mut set| set.all(|x| node_up(&x))); + if has_quorum { + partitions_quorum += 1; + } + if all_ok { + partitions_all_ok += 1; + } + } + + // Determine overall cluster status let status = - if partitions_quorum == partitions.len() && storage_nodes_ok == storage_nodes.len() { + if partitions_all_ok == partitions.len() && storage_nodes_ok == storage_nodes.len() { ClusterHealthStatus::Healthy } else if partitions_quorum == partitions.len() { ClusterHealthStatus::Degraded @@ -543,12 +523,9 @@ impl System { } }; + let hostname = self.local_status.read().unwrap().hostname.clone().unwrap(); if let Err(e) = c - .publish_consul_service( - self.netapp.id, - &self.local_status.load_full().hostname, - rpc_public_addr, - ) + .publish_consul_service(self.netapp.id, &hostname, rpc_public_addr) .await { error!("Error while publishing Consul service: {}", e); @@ -570,38 +547,17 @@ impl System { } }; - if let Err(e) = publish_kubernetes_node( - k, - self.netapp.id, - &self.local_status.load_full().hostname, - rpc_public_addr, - ) - .await + let hostname = self.local_status.read().unwrap().hostname.clone().unwrap(); + if let Err(e) = publish_kubernetes_node(k, self.netapp.id, &hostname, rpc_public_addr).await { error!("Error while publishing node to Kubernetes: {}", e); } } - /// Save network configuration to disc - async fn save_cluster_layout(&self) -> Result<(), Error> { - let ring: Arc = self.ring.borrow().clone(); - self.persist_cluster_layout - .save_async(&ring.layout) - .await - .expect("Cannot save current cluster layout"); - Ok(()) - } - fn update_local_status(&self) { - let mut new_si: NodeStatus = self.local_status.load().as_ref().clone(); - - let ring = self.ring.borrow(); - new_si.cluster_layout_version = ring.layout.version; - new_si.cluster_layout_staging_hash = ring.layout.staging_hash; - - new_si.update_disk_usage(&self.metadata_dir, &self.data_dir, &self.metrics); - - self.local_status.swap(Arc::new(new_si)); + let mut local_status = self.local_status.write().unwrap(); + local_status.layout_digest = self.layout_manager.layout().digest(); + local_status.update_disk_usage(&self.metadata_dir, &self.data_dir); } // --- RPC HANDLERS --- @@ -611,11 +567,6 @@ impl System { Ok(SystemRpc::Ok) } - fn handle_pull_cluster_layout(&self) -> SystemRpc { - let ring = self.ring.borrow().clone(); - SystemRpc::AdvertiseClusterLayout(ring.layout.clone()) - } - fn handle_get_known_nodes(&self) -> SystemRpc { let known_nodes = self.get_known_nodes(); SystemRpc::ReturnKnownNodes(known_nodes) @@ -626,7 +577,7 @@ impl System { from: Uuid, info: &NodeStatus, ) -> Result { - let local_info = self.local_status.load(); + let local_info = self.local_status.read().unwrap(); if local_info.replication_factor < info.replication_factor { error!("Some node have a higher replication factor ({}) than this one ({}). This is not supported and will lead to data corruption. Shutting down for safety.", @@ -635,11 +586,10 @@ impl System { std::process::exit(1); } - if info.cluster_layout_version > local_info.cluster_layout_version - || info.cluster_layout_staging_hash != local_info.cluster_layout_staging_hash - { - tokio::spawn(self.clone().pull_cluster_layout(from)); - } + self.layout_manager + .handle_advertise_status(from, &info.layout_digest); + + drop(local_info); self.node_status .write() @@ -649,90 +599,44 @@ impl System { Ok(SystemRpc::Ok) } - async fn handle_advertise_cluster_layout( - self: &Arc, - adv: &ClusterLayout, - ) -> Result { - if adv.replication_factor != self.replication_factor { - let msg = format!( - "Received a cluster layout from another node with replication factor {}, which is different from what we have in our configuration ({}). Discarding the cluster layout we received.", - adv.replication_factor, - self.replication_factor - ); - error!("{}", msg); - return Err(Error::Message(msg)); - } - - let update_ring = self.update_ring.lock().await; - let mut layout: ClusterLayout = self.ring.borrow().layout.clone(); - - let prev_layout_check = layout.check(); - if layout.merge(adv) { - if prev_layout_check && !layout.check() { - error!("New cluster layout is invalid, discarding."); - return Err(Error::Message( - "New cluster layout is invalid, discarding.".into(), - )); - } - - let ring = Ring::new(layout.clone(), self.replication_factor); - update_ring.send(Arc::new(ring))?; - drop(update_ring); - - let self2 = self.clone(); - tokio::spawn(async move { - if let Err(e) = self2 - .rpc - .broadcast( - &self2.system_endpoint, - SystemRpc::AdvertiseClusterLayout(layout), - RequestStrategy::with_priority(PRIO_HIGH), - ) - .await - { - warn!("Error while broadcasting new cluster layout: {}", e); - } - }); - - self.save_cluster_layout().await?; - } - - Ok(SystemRpc::Ok) - } - async fn status_exchange_loop(&self, mut stop_signal: watch::Receiver) { while !*stop_signal.borrow() { - let restart_at = tokio::time::sleep(STATUS_EXCHANGE_INTERVAL); + let restart_at = Instant::now() + STATUS_EXCHANGE_INTERVAL; + // Update local node status that is exchanged. self.update_local_status(); - let local_status: NodeStatus = self.local_status.load().as_ref().clone(); + + let local_status: NodeStatus = self.local_status.read().unwrap().clone(); let _ = self - .rpc + .rpc_helper() .broadcast( &self.system_endpoint, SystemRpc::AdvertiseStatus(local_status), - RequestStrategy::with_priority(PRIO_HIGH), + RequestStrategy::with_priority(PRIO_HIGH) + .with_custom_timeout(STATUS_EXCHANGE_INTERVAL), ) .await; select! { - _ = restart_at.fuse() => {}, - _ = stop_signal.changed().fuse() => {}, + _ = tokio::time::sleep_until(restart_at.into()) => {}, + _ = stop_signal.changed() => {}, } } } async fn discovery_loop(self: &Arc, mut stop_signal: watch::Receiver) { while !*stop_signal.borrow() { - let not_configured = !self.ring.borrow().layout.check(); - let no_peers = self.fullmesh.get_peer_list().len() < self.replication_factor; - let expected_n_nodes = self.ring.borrow().layout.num_nodes(); - let bad_peers = self - .fullmesh + let n_connected = self + .peering .get_peer_list() .iter() .filter(|p| p.is_up()) - .count() != expected_n_nodes; + .count(); + + let not_configured = !self.cluster_layout().is_check_ok(); + let no_peers = n_connected < self.replication_factor.into(); + let expected_n_nodes = self.cluster_layout().all_nodes().len(); + let bad_peers = n_connected != expected_n_nodes; if not_configured || no_peers || bad_peers { info!("Doing a bootstrap/discovery step (not_configured: {}, no_peers: {}, bad_peers: {})", not_configured, no_peers, bad_peers); @@ -779,6 +683,14 @@ impl System { } } + if !not_configured && !no_peers { + // If the layout is configured, and we already have some connections + // to other nodes in the cluster, we can skip trying to connect to + // nodes that are not in the cluster layout. + let layout = self.cluster_layout(); + ping_list.retain(|(id, _)| layout.all_nodes().contains(&(*id).into())); + } + for (node_id, node_addr) in ping_list { let self2 = self.clone(); tokio::spawn(async move { @@ -799,10 +711,9 @@ impl System { #[cfg(feature = "kubernetes-discovery")] tokio::spawn(self.clone().advertise_to_kubernetes()); - let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL); select! { - _ = restart_at.fuse() => {}, - _ = stop_signal.changed().fuse() => {}, + _ = tokio::time::sleep(DISCOVERY_INTERVAL) => {}, + _ = stop_signal.changed() => {}, } } } @@ -811,10 +722,13 @@ impl System { // Prepare new peer list to save to file // It is a vec of tuples (node ID as Uuid, node SocketAddr) let mut peer_list = self - .fullmesh + .peering .get_peer_list() .iter() - .map(|n| (n.id.into(), n.addr)) + .filter_map(|n| match n.state { + PeerConnState::Connected { addr } => Some((n.id.into(), addr)), + _ => None, + }) .collect::>(); // Before doing it, we read the current peer list file (if it exists) @@ -832,48 +746,48 @@ impl System { .save_async(&PeerList(peer_list)) .await } - - async fn pull_cluster_layout(self: Arc, peer: Uuid) { - let resp = self - .rpc - .call( - &self.system_endpoint, - peer, - SystemRpc::PullClusterLayout, - RequestStrategy::with_priority(PRIO_HIGH), - ) - .await; - if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp { - let _: Result<_, _> = self.handle_advertise_cluster_layout(&layout).await; - } - } } -#[async_trait] impl EndpointHandler for System { async fn handle(self: &Arc, msg: &SystemRpc, from: NodeID) -> Result { match msg { + // ---- system functions -> System ---- SystemRpc::Connect(node) => self.handle_connect(node).await, - SystemRpc::PullClusterLayout => Ok(self.handle_pull_cluster_layout()), SystemRpc::AdvertiseStatus(adv) => self.handle_advertise_status(from.into(), adv).await, - SystemRpc::AdvertiseClusterLayout(adv) => { - self.clone().handle_advertise_cluster_layout(adv).await - } SystemRpc::GetKnownNodes => Ok(self.handle_get_known_nodes()), + + // ---- layout functions -> LayoutManager ---- + SystemRpc::PullClusterLayout => Ok(self.layout_manager.handle_pull_cluster_layout()), + SystemRpc::AdvertiseClusterLayout(adv) => { + self.layout_manager + .handle_advertise_cluster_layout(adv) + .await + } + SystemRpc::PullClusterLayoutTrackers => { + Ok(self.layout_manager.handle_pull_cluster_layout_trackers()) + } + SystemRpc::AdvertiseClusterLayoutTrackers(adv) => { + self.layout_manager + .handle_advertise_cluster_layout_trackers(adv) + .await + } + + // ---- other -> Error ---- m => Err(Error::unexpected_rpc_message(m)), } } } impl NodeStatus { - fn initial(replication_factor: usize, layout: &ClusterLayout) -> Self { + fn initial(replication_factor: ReplicationFactor, layout_manager: &LayoutManager) -> Self { NodeStatus { - hostname: gethostname::gethostname() - .into_string() - .unwrap_or_else(|_| "".to_string()), - replication_factor, - cluster_layout_version: layout.version, - cluster_layout_staging_hash: layout.staging_hash, + hostname: Some( + gethostname::gethostname() + .into_string() + .unwrap_or_else(|_| "".to_string()), + ), + replication_factor: replication_factor.into(), + layout_digest: layout_manager.layout().digest(), meta_disk_avail: None, data_disk_avail: None, } @@ -881,59 +795,146 @@ impl NodeStatus { fn unknown() -> Self { NodeStatus { - hostname: "?".to_string(), + hostname: None, replication_factor: 0, - cluster_layout_version: 0, - cluster_layout_staging_hash: Hash::from([0u8; 32]), + layout_digest: Default::default(), meta_disk_avail: None, data_disk_avail: None, } } - fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &Path, metrics: &SystemMetrics) { - use systemstat::{Platform, System}; - let mounts = System::new().mounts().unwrap_or_default(); + fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &DataDirEnum) { + use nix::sys::statvfs::statvfs; - let mount_avail = |path: &Path| { - mounts - .iter() - .filter(|x| path.starts_with(&x.fs_mounted_on)) - .max_by_key(|x| x.fs_mounted_on.len()) - .map(|x| (x.avail.as_u64(), x.total.as_u64())) + // The HashMap used below requires a filesystem identifier from statfs (instead of statvfs) on FreeBSD, as + // FreeBSD's statvfs filesystem identifier is "not meaningful in this implementation" (man 3 statvfs). + + #[cfg(target_os = "freebsd")] + let get_filesystem_id = |path: &Path| match nix::sys::statfs::statfs(path) { + Ok(fs) => Some(fs.filesystem_id()), + Err(_) => None, }; - self.meta_disk_avail = mount_avail(meta_dir); - self.data_disk_avail = mount_avail(data_dir); + let mount_avail = |path: &Path| match statvfs(path) { + Ok(x) => { + let avail = x.blocks_available() as u64 * x.fragment_size() as u64; + let total = x.blocks() as u64 * x.fragment_size() as u64; + Some((x.filesystem_id(), avail, total)) + } + Err(_) => None, + }; - if let Some((avail, total)) = self.meta_disk_avail { - metrics - .values - .meta_disk_avail - .store(avail, Ordering::Relaxed); - metrics - .values - .meta_disk_total - .store(total, Ordering::Relaxed); - } - if let Some((avail, total)) = self.data_disk_avail { - metrics - .values - .data_disk_avail - .store(avail, Ordering::Relaxed); - metrics - .values - .data_disk_total - .store(total, Ordering::Relaxed); - } + self.meta_disk_avail = mount_avail(meta_dir).map(|(_, a, t)| (a, t)); + + self.data_disk_avail = match data_dir { + DataDirEnum::Single(dir) => mount_avail(dir).map(|(_, a, t)| (a, t)), + DataDirEnum::Multiple(dirs) => (|| { + // TODO: more precise calculation that takes into account + // how data is going to be spread among partitions + let mut mounts = HashMap::new(); + for dir in dirs.iter() { + if dir.capacity.is_none() { + continue; + } + + #[cfg(not(target_os = "freebsd"))] + match mount_avail(&dir.path) { + Some((fsid, avail, total)) => { + mounts.insert(fsid, (avail, total)); + } + None => return None, + } + + #[cfg(target_os = "freebsd")] + match get_filesystem_id(&dir.path) { + Some(fsid) => match mount_avail(&dir.path) { + Some((_, avail, total)) => { + mounts.insert(fsid, (avail, total)); + } + None => return None, + }, + None => return None, + } + } + Some( + mounts + .into_iter() + .fold((0, 0), |(x, y), (_, (a, b))| (x + a, y + b)), + ) + })(), + }; } } -fn get_default_ip() -> Option { +/// Obtain the list of currently available IP addresses on all non-loopback +/// interfaces, optionally filtering them to be inside a given IpNet. +fn get_default_ip(filter_ipnet: Option) -> Option { pnet_datalink::interfaces() - .iter() - .find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) - .and_then(|e| e.ips.first()) - .map(|a| a.ip()) + .into_iter() + // filter down and loopback interfaces + .filter(|i| i.is_up() && !i.is_loopback()) + // get all IPs + .flat_map(|e| e.ips) + // optionally, filter to be inside filter_ipnet + .find(|ipn| { + filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none() + }) + .map(|ipn| ipn.ip()) +} + +fn get_rpc_public_addr(config: &Config) -> Option { + match &config.rpc_public_addr { + Some(a_str) => { + use std::net::ToSocketAddrs; + match a_str.to_socket_addrs() { + Err(e) => { + error!( + "Cannot resolve rpc_public_addr {} from config file: {}.", + a_str, e + ); + None + } + Ok(a) => { + let a = a.collect::>(); + if a.is_empty() { + error!("rpc_public_addr {} resolve to no known IP address", a_str); + } + if a.len() > 1 { + warn!("Multiple possible resolutions for rpc_public_addr: {:?}. Taking the first one.", a); + } + a.into_iter().next() + } + } + } + None => { + // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`. + let filter_subnet: Option = config + .rpc_public_addr_subnet + .as_ref() + .and_then(|filter_subnet_str| match filter_subnet_str.parse::() { + Ok(filter_subnet) => { + let filter_subnet_trunc = filter_subnet.trunc(); + if filter_subnet_trunc != filter_subnet { + warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc()); + } + Some(filter_subnet_trunc) + } + Err(e) => { + panic!( + "Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.", + filter_subnet_str, e + ); + } + }); + + let addr = get_default_ip(filter_subnet) + .map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); + if let Some(a) = addr { + warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); + } + addr + } + } } async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> { @@ -958,7 +959,7 @@ async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> { fn connect_error_message( addr: SocketAddr, pubkey: ed25519::PublicKey, - e: netapp::error::Error, + e: garage_net::error::Error, ) -> String { format!("Error establishing RPC connection to remote node: {}@{}.\nThis can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret.\n{}", hex::encode(pubkey), addr, e) } diff --git a/src/rpc/system_metrics.rs b/src/rpc/system_metrics.rs index af81b71f..a64daec8 100644 --- a/src/rpc/system_metrics.rs +++ b/src/rpc/system_metrics.rs @@ -1,32 +1,57 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; use opentelemetry::{global, metrics::*, KeyValue}; +use crate::system::{ClusterHealthStatus, System}; + /// TableMetrics reference all counter used for metrics pub struct SystemMetrics { + // Static values pub(crate) _garage_build_info: ValueObserver, pub(crate) _replication_factor: ValueObserver, + + // Disk space values from System::local_status pub(crate) _disk_avail: ValueObserver, pub(crate) _disk_total: ValueObserver, - pub(crate) values: Arc, -} -#[derive(Default)] -pub struct SystemMetricsValues { - pub(crate) data_disk_total: AtomicU64, - pub(crate) data_disk_avail: AtomicU64, - pub(crate) meta_disk_total: AtomicU64, - pub(crate) meta_disk_avail: AtomicU64, + // Health report from System::health() + pub(crate) _cluster_healthy: ValueObserver, + pub(crate) _cluster_available: ValueObserver, + pub(crate) _known_nodes: ValueObserver, + pub(crate) _connected_nodes: ValueObserver, + pub(crate) _storage_nodes: ValueObserver, + pub(crate) _storage_nodes_ok: ValueObserver, + pub(crate) _partitions: ValueObserver, + pub(crate) _partitions_quorum: ValueObserver, + pub(crate) _partitions_all_ok: ValueObserver, + + // Status report for individual cluster nodes + pub(crate) _layout_node_connected: ValueObserver, + pub(crate) _layout_node_disconnected_time: ValueObserver, } impl SystemMetrics { - pub fn new(replication_factor: usize) -> Self { + pub fn new(system: Arc) -> Self { let meter = global::meter("garage_system"); - let values = Arc::new(SystemMetricsValues::default()); - let values1 = values.clone(); - let values2 = values.clone(); + + let health_cache = RwLock::new((Instant::now(), system.health())); + let system2 = system.clone(); + let get_health = Arc::new(move || { + { + let cache = health_cache.read().unwrap(); + if cache.0 > Instant::now() - Duration::from_secs(1) { + return cache.1; + } + } + + let health = system2.health(); + *health_cache.write().unwrap() = (Instant::now(), health); + health + }); + Self { + // Static values _garage_build_info: meter .u64_value_observer("garage_build_info", move |observer| { observer.observe( @@ -39,39 +64,239 @@ impl SystemMetrics { }) .with_description("Garage build info") .init(), - _replication_factor: meter - .u64_value_observer("garage_replication_factor", move |observer| { - observer.observe(replication_factor as u64, &[]) + _replication_factor: { + let replication_factor = system.replication_factor; + meter + .u64_value_observer("garage_replication_factor", move |observer| { + observer.observe(replication_factor.replication_factor() as u64, &[]) + }) + .with_description("Garage replication factor setting") + .init() + }, + + // Disk space values from System::local_status + _disk_avail: { + let system = system.clone(); + meter + .u64_value_observer("garage_local_disk_avail", move |observer| { + let st = system.local_status.read().unwrap(); + if let Some((avail, _total)) = st.data_disk_avail { + observer.observe(avail, &[KeyValue::new("volume", "data")]); + } + if let Some((avail, _total)) = st.meta_disk_avail { + observer.observe(avail, &[KeyValue::new("volume", "metadata")]); + } + }) + .with_description("Garage available disk space on each node") + .init() + }, + _disk_total: { + let system = system.clone(); + meter + .u64_value_observer("garage_local_disk_total", move |observer| { + let st = system.local_status.read().unwrap(); + if let Some((_avail, total)) = st.data_disk_avail { + observer.observe(total, &[KeyValue::new("volume", "data")]); + } + if let Some((_avail, total)) = st.meta_disk_avail { + observer.observe(total, &[KeyValue::new("volume", "metadata")]); + } + }) + .with_description("Garage total disk space on each node") + .init() + }, + + // Health report from System::() + _cluster_healthy: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_healthy", move |observer| { + let h = get_health(); + if h.status == ClusterHealthStatus::Healthy { + observer.observe(1, &[]); + } else { + observer.observe(0, &[]); + } + }) + .with_description("Whether all storage nodes are connected") + .init() + }, + _cluster_available: { + let get_health = get_health.clone(); + meter.u64_value_observer("cluster_available", move |observer| { + let h = get_health(); + if h.status != ClusterHealthStatus::Unavailable { + observer.observe(1, &[]); + } else { + observer.observe(0, &[]); + } }) - .with_description("Garage replication factor setting") - .init(), - _disk_avail: meter - .u64_value_observer("garage_local_disk_avail", move |observer| { - match values1.data_disk_avail.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "data")]), - }; - match values1.meta_disk_avail.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), - }; - }) - .with_description("Garage available disk space on each node") - .init(), - _disk_total: meter - .u64_value_observer("garage_local_disk_total", move |observer| { - match values2.data_disk_total.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "data")]), - }; - match values2.meta_disk_total.load(Ordering::Relaxed) { - 0 => (), - x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), - }; - }) - .with_description("Garage total disk space on each node") - .init(), - values, + .with_description("Whether all requests can be served, even if some storage nodes are disconnected") + .init() + }, + _known_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_known_nodes", move |observer| { + let h = get_health(); + observer.observe(h.known_nodes as u64, &[]); + }) + .with_description("Number of nodes already seen once in the cluster") + .init() + }, + _connected_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_connected_nodes", move |observer| { + let h = get_health(); + observer.observe(h.connected_nodes as u64, &[]); + }) + .with_description("Number of nodes currently connected") + .init() + }, + _storage_nodes: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_storage_nodes", move |observer| { + let h = get_health(); + observer.observe(h.storage_nodes as u64, &[]); + }) + .with_description("Number of storage nodes declared in the current layout") + .init() + }, + _storage_nodes_ok: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_storage_nodes_ok", move |observer| { + let h = get_health(); + observer.observe(h.storage_nodes_ok as u64, &[]); + }) + .with_description("Number of storage nodes currently connected") + .init() + }, + _partitions: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions", move |observer| { + let h = get_health(); + observer.observe(h.partitions as u64, &[]); + }) + .with_description("Number of partitions in the layout") + .init() + }, + _partitions_quorum: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions_quorum", move |observer| { + let h = get_health(); + observer.observe(h.partitions_quorum as u64, &[]); + }) + .with_description( + "Number of partitions for which we have a quorum of connected nodes", + ) + .init() + }, + _partitions_all_ok: { + let get_health = get_health.clone(); + meter + .u64_value_observer("cluster_partitions_all_ok", move |observer| { + let h = get_health(); + observer.observe(h.partitions_all_ok as u64, &[]); + }) + .with_description( + "Number of partitions for which all storage nodes are connected", + ) + .init() + }, + + // Status report for individual cluster nodes + _layout_node_connected: { + let system = system.clone(); + meter + .u64_value_observer("cluster_layout_node_connected", move |observer| { + let layout = system.cluster_layout(); + let nodes = system.get_known_nodes(); + for id in layout.all_nodes().iter() { + let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; + if let Some(role) = + layout.current().roles.get(id).and_then(|r| r.0.as_ref()) + { + kv.push(KeyValue::new("role_zone", role.zone.clone())); + match role.capacity { + Some(cap) => { + kv.push(KeyValue::new("role_capacity", cap as i64)); + kv.push(KeyValue::new("role_gateway", 0)); + } + None => { + kv.push(KeyValue::new("role_gateway", 1)); + } + } + } + + let value; + if let Some(node) = nodes.iter().find(|n| n.id == *id) { + // TODO: if we add address and hostname, and those change, we + // get duplicate metrics, due to bad otel aggregation :( + // Can probably be fixed when we upgrade opentelemetry + // kv.push(KeyValue::new("address", node.addr.to_string())); + // kv.push(KeyValue::new( + // "hostname", + // node.status.hostname.clone(), + // )); + value = if node.is_up { 1 } else { 0 }; + } else { + value = 0; + } + + observer.observe(value, &kv); + } + }) + .with_description("Connection status for nodes in the cluster layout") + .init() + }, + _layout_node_disconnected_time: { + let system = system.clone(); + meter + .u64_value_observer("cluster_layout_node_disconnected_time", move |observer| { + let layout = system.cluster_layout(); + let nodes = system.get_known_nodes(); + for id in layout.all_nodes().iter() { + let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; + if let Some(role) = + layout.current().roles.get(id).and_then(|r| r.0.as_ref()) + { + kv.push(KeyValue::new("role_zone", role.zone.clone())); + match role.capacity { + Some(cap) => { + kv.push(KeyValue::new("role_capacity", cap as i64)); + kv.push(KeyValue::new("role_gateway", 0)); + } + None => { + kv.push(KeyValue::new("role_gateway", 1)); + } + } + } + + if let Some(node) = nodes.iter().find(|n| n.id == *id) { + // TODO: see comment above + // kv.push(KeyValue::new("address", node.addr.to_string())); + // kv.push(KeyValue::new( + // "hostname", + // node.status.hostname.clone(), + // )); + if node.is_up { + observer.observe(0, &kv); + } else if let Some(secs) = node.last_seen_secs_ago { + observer.observe(secs, &kv); + } + } + } + }) + .with_description( + "Time (in seconds) since last connection to nodes in the cluster layout", + ) + .init() + }, } } } diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index c794c924..478dbd18 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,23 +14,22 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.2", path = "../db" } -garage_rpc = { version = "0.8.2", path = "../rpc" } -garage_util = { version = "0.8.2", path = "../util" } +garage_db.workspace = true +garage_rpc.workspace = true +garage_util.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true -async-trait = "0.1.7" -arc-swap = "1.0" -bytes = "1.0" -hex = "0.4" -hexdump = "0.1" -tracing = "0.1" -rand = "0.8" +async-trait.workspace = true +arc-swap.workspace = true +hex.workspace = true +hexdump.workspace = true +tracing.workspace = true +rand.workspace = true -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" +serde.workspace = true +serde_bytes.workspace = true -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true diff --git a/src/table/data.rs b/src/table/data.rs index 26cc3a5a..1d0308ce 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -6,7 +6,6 @@ use serde_bytes::ByteBuf; use tokio::sync::Notify; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; use garage_util::data::*; use garage_util::error::*; @@ -34,9 +33,9 @@ pub struct TableData { pub(crate) merkle_todo_notify: Notify, pub(crate) insert_queue: db::Tree, - pub(crate) insert_queue_notify: Notify, + pub(crate) insert_queue_notify: Arc, - pub(crate) gc_todo: CountedTree, + pub(crate) gc_todo: db::Tree, pub(crate) metrics: TableMetrics, } @@ -44,24 +43,23 @@ pub struct TableData { impl TableData { pub fn new(system: Arc, instance: F, replication: R, db: &db::Db) -> Arc { let store = db - .open_tree(&format!("{}:table", F::TABLE_NAME)) + .open_tree(format!("{}:table", F::TABLE_NAME)) .expect("Unable to open DB tree"); let merkle_tree = db - .open_tree(&format!("{}:merkle_tree", F::TABLE_NAME)) + .open_tree(format!("{}:merkle_tree", F::TABLE_NAME)) .expect("Unable to open DB Merkle tree tree"); let merkle_todo = db - .open_tree(&format!("{}:merkle_todo", F::TABLE_NAME)) + .open_tree(format!("{}:merkle_todo", F::TABLE_NAME)) .expect("Unable to open DB Merkle TODO tree"); let insert_queue = db - .open_tree(&format!("{}:insert_queue", F::TABLE_NAME)) + .open_tree(format!("{}:insert_queue", F::TABLE_NAME)) .expect("Unable to open insert queue DB tree"); let gc_todo = db - .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) + .open_tree(format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open GC DB tree"); - let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2"); let metrics = TableMetrics::new( F::TABLE_NAME, @@ -80,7 +78,7 @@ impl TableData { merkle_todo, merkle_todo_notify: Notify::new(), insert_queue, - insert_queue_notify: Notify::new(), + insert_queue_notify: Arc::new(Notify::new()), gc_todo, metrics, }) @@ -90,7 +88,7 @@ impl TableData { pub fn read_entry(&self, p: &F::P, s: &F::S) -> Result, Error> { let tree_key = self.tree_key(p, s); - if let Some(bytes) = self.store.get(&tree_key)? { + if let Some(bytes) = self.store.get(tree_key)? { Ok(Some(ByteBuf::from(bytes.to_vec()))) } else { Ok(None) @@ -132,10 +130,10 @@ impl TableData { } } - fn read_range_aux<'a>( + fn read_range_aux( &self, partition_hash: Hash, - range: db::ValueIter<'a>, + range: db::ValueIter, filter: &Option, limit: usize, ) -> Result>, Error> { @@ -203,14 +201,14 @@ impl TableData { ) -> Result, Error> { let tree_key = self.tree_key(partition_key, sort_key); - let changed = self.store.db().transaction(|mut tx| { + let changed = self.store.db().transaction(|tx| { let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? { Some(old_bytes) => { let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?; - let new_entry = update_fn(&mut tx, Some(old_entry.clone()))?; + let new_entry = update_fn(tx, Some(old_entry.clone()))?; (Some(old_entry), Some(old_bytes), new_entry) } - None => (None, None, update_fn(&mut tx, None)?), + None => (None, None, update_fn(tx, None)?), }; // Changed can be true in two scenarios @@ -233,7 +231,7 @@ impl TableData { tx.insert(&self.store, &tree_key, new_bytes)?; self.instance - .updated(&mut tx, old_entry.as_ref(), Some(&new_entry))?; + .updated(tx, old_entry.as_ref(), Some(&new_entry))?; Ok(Some((new_entry, new_bytes_hash))) } else { @@ -254,7 +252,8 @@ impl TableData { // of the GC algorithm, as in all cases GC is suspended if // any node of the partition is unavailable. let pk_hash = Hash::try_from(&tree_key[..32]).unwrap(); - let nodes = self.replication.write_nodes(&pk_hash); + // TODO: this probably breaks when the layout changes + let nodes = self.replication.storage_nodes(&pk_hash); if nodes.first() == Some(&self.system.id) { GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?; } @@ -270,14 +269,14 @@ impl TableData { let removed = self .store .db() - .transaction(|mut tx| match tx.get(&self.store, k)? { + .transaction(|tx| match tx.get(&self.store, k)? { Some(cur_v) if cur_v == v => { let old_entry = self.decode_entry(v).map_err(db::TxError::Abort)?; tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; - self.instance.updated(&mut tx, Some(&old_entry), None)?; + self.instance.updated(tx, Some(&old_entry), None)?; Ok(true) } _ => Ok(false), @@ -298,14 +297,14 @@ impl TableData { let removed = self .store .db() - .transaction(|mut tx| match tx.get(&self.store, k)? { + .transaction(|tx| match tx.get(&self.store, k)? { Some(cur_v) if blake2sum(&cur_v[..]) == vhash => { let old_entry = self.decode_entry(&cur_v[..]).map_err(db::TxError::Abort)?; tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; - self.instance.updated(&mut tx, Some(&old_entry), None)?; + self.instance.updated(tx, Some(&old_entry), None)?; Ok(true) } _ => Ok(false), @@ -339,7 +338,9 @@ impl TableData { .map_err(Error::RmpEncode) .map_err(db::TxError::Abort)?; tx.insert(&self.insert_queue, &tree_key, new_entry)?; - self.insert_queue_notify.notify_one(); + + let notif = self.insert_queue_notify.clone(); + tx.on_commit(move || notif.notify_one()); Ok(()) } @@ -347,9 +348,7 @@ impl TableData { // ---- Utility functions ---- pub fn tree_key(&self, p: &F::P, s: &F::S) -> Vec { - let mut ret = p.hash().to_vec(); - ret.extend(s.sort_key()); - ret + [p.hash().as_slice(), s.sort_key()].concat() } pub fn decode_entry(&self, bytes: &[u8]) -> Result { @@ -368,7 +367,7 @@ impl TableData { } } - pub fn gc_todo_len(&self) -> Result { - Ok(self.gc_todo.len()) + pub fn gc_todo_approximate_len(&self) -> Result { + Ok(self.gc_todo.approximate_len()?) } } diff --git a/src/table/gc.rs b/src/table/gc.rs index 5b9124a7..1f30bd76 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -4,13 +4,14 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; + use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use futures::future::join_all; use tokio::sync::watch; -use garage_db::counted_tree_hack::CountedTree; +use garage_db as db; use garage_util::background::*; use garage_util::data::*; @@ -152,7 +153,7 @@ impl TableGc { let mut partitions = HashMap::new(); for entry in entries { let pkh = Hash::try_from(&entry.key[..32]).unwrap(); - let mut nodes = self.data.replication.write_nodes(&pkh); + let mut nodes = self.data.replication.storage_nodes(&pkh); nodes.retain(|x| *x != self.system.id); nodes.sort(); @@ -227,10 +228,10 @@ impl TableGc { // GC'ing is not a critical function of the system, so it's not a big // deal if we can't do it right now. self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &nodes[..], + &nodes, GcRpc::Update(updates), RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()), ) @@ -248,31 +249,30 @@ impl TableGc { // it means that the garbage collection wasn't completed and has // to be retried later. self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &nodes[..], + &nodes, GcRpc::DeleteIfEqualHash(deletes), RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()), ) .await .err_context("GC: remote delete tombstones")?; - // GC has been successfull for all of these entries. + // GC has been successful for all of these entries. // We now remove them all from our local table and from the GC todo list. for item in items { self.data .delete_if_equal_hash(&item.key[..], item.value_hash) .err_context("GC: local delete tombstones")?; item.remove_if_equal(&self.data.gc_todo) - .err_context("GC: remove from todo list after successfull GC")?; + .err_context("GC: remove from todo list after successful GC")?; } Ok(()) } } -#[async_trait] impl EndpointHandler for TableGc { async fn handle(self: &Arc, message: &GcRpc, _from: NodeID) -> Result { match message { @@ -313,7 +313,7 @@ impl Worker for GcWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64), + queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64), ..Default::default() } } @@ -334,9 +334,9 @@ impl Worker for GcWorker { } } -/// An entry stored in the gc_todo Sled tree associated with the table +/// An entry stored in the gc_todo db tree associated with the table /// Contains helper function for parsing, saving, and removing -/// such entry in Sled +/// such entry in the db /// /// Format of an entry: /// - key = 8 bytes: timestamp of tombstone @@ -353,7 +353,7 @@ pub(crate) struct GcTodoEntry { } impl GcTodoEntry { - /// Creates a new GcTodoEntry (not saved in Sled) from its components: + /// Creates a new GcTodoEntry (not saved in the db) from its components: /// the key of an entry in the table, and the hash of the associated /// serialized value pub(crate) fn new(key: Vec, value_hash: Hash) -> Self { @@ -376,22 +376,24 @@ impl GcTodoEntry { } /// Saves the GcTodoEntry in the gc_todo tree - pub(crate) fn save(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { + pub(crate) fn save(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { gc_todo_tree.insert(self.todo_table_key(), self.value_hash.as_slice())?; Ok(()) } /// Removes the GcTodoEntry from the gc_todo tree if the /// hash of the serialized value is the same here as in the tree. - /// This is usefull to remove a todo entry only under the condition + /// This is useful to remove a todo entry only under the condition /// that it has not changed since the time it was read, i.e. /// what we have to do is still the same - pub(crate) fn remove_if_equal(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { - gc_todo_tree.compare_and_swap::<_, _, &[u8]>( - &self.todo_table_key(), - Some(self.value_hash), - None, - )?; + pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { + gc_todo_tree.db().transaction(|txn| { + let key = self.todo_table_key(); + if txn.get(gc_todo_tree, &key)?.as_deref() == Some(self.value_hash.as_slice()) { + txn.remove(gc_todo_tree, &key)?; + } + Ok(()) + })?; Ok(()) } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index e86d0251..7ba1f007 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -13,7 +13,7 @@ use garage_util::data::*; use garage_util::encode::{nonversioned_decode, nonversioned_encode}; use garage_util::error::Error; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use crate::data::*; use crate::replication::*; @@ -31,14 +31,14 @@ pub struct MerkleUpdater { // - value = the hash of the full serialized item, if present, // or an empty vec if item is absent (deleted) // Fields in data: - // pub(crate) merkle_todo: sled::Tree, + // pub(crate) merkle_todo: db::Tree, // pub(crate) merkle_todo_notify: Notify, // Content of the merkle tree: items where // - key = .bytes() for MerkleNodeKey // - value = serialization of a MerkleNode, assumed to be MerkleNode::empty if not found // Field in data: - // pub(crate) merkle_tree: sled::Tree, + // pub(crate) merkle_tree: db::Tree, empty_node_hash: Hash, } @@ -108,9 +108,9 @@ impl MerkleUpdater { self.data .merkle_tree .db() - .transaction(|mut tx| self.update_item_rec(&mut tx, k, &khash, &key, new_vhash))?; + .transaction(|tx| self.update_item_rec(tx, k, &khash, &key, new_vhash))?; - let deleted = self.data.merkle_todo.db().transaction(|mut tx| { + let deleted = self.data.merkle_todo.db().transaction(|tx| { let remove = matches!(tx.get(&self.data.merkle_todo, k)?, Some(ov) if ov == vhash_by); if remove { tx.remove(&self.data.merkle_todo, k)?; @@ -287,16 +287,12 @@ impl MerkleUpdater { MerkleNode::decode_opt(&ent) } - pub fn merkle_tree_len(&self) -> Result { - Ok(self.data.merkle_tree.len()?) + pub fn merkle_tree_approximate_len(&self) -> Result { + Ok(self.data.merkle_tree.approximate_len()?) } - pub fn merkle_tree_fast_len(&self) -> Result, Error> { - Ok(self.data.merkle_tree.fast_len()?) - } - - pub fn todo_len(&self) -> Result { - Ok(self.data.merkle_todo.len()?) + pub fn todo_approximate_len(&self) -> Result { + Ok(self.data.merkle_todo.approximate_len()?) } } @@ -310,7 +306,7 @@ impl Worker for MerkleWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.0.todo_len().unwrap_or(0) as u64), + queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64), ..Default::default() } } diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 8318a84f..78593202 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,7 +1,6 @@ use opentelemetry::{global, metrics::*, KeyValue}; use garage_db as db; -use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct TableMetrics { @@ -27,7 +26,7 @@ impl TableMetrics { store: db::Tree, merkle_tree: db::Tree, merkle_todo: db::Tree, - gc_todo: CountedTree, + gc_todo: db::Tree, ) -> Self { let meter = global::meter(table_name); TableMetrics { @@ -35,9 +34,9 @@ impl TableMetrics { .u64_value_observer( "table.size", move |observer| { - if let Ok(Some(v)) = store.fast_len() { + if let Ok(value) = store.approximate_len() { observer.observe( - v as u64, + value as u64, &[KeyValue::new("table_name", table_name)], ); } @@ -49,9 +48,9 @@ impl TableMetrics { .u64_value_observer( "table.merkle_tree_size", move |observer| { - if let Ok(Some(v)) = merkle_tree.fast_len() { + if let Ok(value) = merkle_tree.approximate_len() { observer.observe( - v as u64, + value as u64, &[KeyValue::new("table_name", table_name)], ); } @@ -63,7 +62,7 @@ impl TableMetrics { .u64_value_observer( "table.merkle_updater_todo_queue_length", move |observer| { - if let Ok(v) = merkle_todo.len() { + if let Ok(v) = merkle_todo.approximate_len() { observer.observe( v as u64, &[KeyValue::new("table_name", table_name)], @@ -77,10 +76,12 @@ impl TableMetrics { .u64_value_observer( "table.gc_todo_queue_length", move |observer| { - observer.observe( - gc_todo.len() as u64, - &[KeyValue::new("table_name", table_name)], - ); + if let Ok(value) = gc_todo.approximate_len() { + observer.observe( + value as u64, + &[KeyValue::new("table_name", table_name)], + ); + } }, ) .with_description("Table garbage collector TODO queue length") diff --git a/src/table/queue.rs b/src/table/queue.rs index 0857209b..7ef1f16e 100644 --- a/src/table/queue.rs +++ b/src/table/queue.rs @@ -12,7 +12,7 @@ use crate::replication::*; use crate::schema::*; use crate::table::*; -const BATCH_SIZE: usize = 100; +const BATCH_SIZE: usize = 1024; pub(crate) struct InsertQueueWorker(pub(crate) Arc>) where @@ -27,7 +27,7 @@ impl Worker for InsertQueueWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64), + queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64), ..Default::default() } } @@ -53,7 +53,7 @@ impl Worker for InsertQueueWorker { self.0.insert_many(values).await?; - self.0.data.insert_queue.db().transaction(|mut tx| { + self.0.data.insert_queue.db().transaction(|tx| { for (k, v) in kv_pairs.iter() { if let Some(v2) = tx.get(&self.0.data.insert_queue, k)? { if &v2 == v { diff --git a/src/table/replication/fullcopy.rs b/src/table/replication/fullcopy.rs index 18682ace..1e52bb47 100644 --- a/src/table/replication/fullcopy.rs +++ b/src/table/replication/fullcopy.rs @@ -1,24 +1,36 @@ use std::sync::Arc; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_rpc::system::System; use garage_util::data::*; use crate::replication::*; +// TODO: find a way to track layout changes for this as well +// The hard thing is that this data is stored also on gateway nodes, +// whereas sharded data is stored only on non-Gateway nodes (storage nodes) +// Also we want to be more tolerant to failures of gateways so we don't +// want to do too much holding back of data when progress of gateway +// nodes is not reported in the layout history's ack/sync/sync_ack maps. + /// Full replication schema: all nodes store everything -/// Writes are disseminated in an epidemic manner in the network /// Advantage: do all reads locally, extremely fast /// Inconvenient: only suitable to reasonably small tables +/// Inconvenient: if some writes fail, nodes will read outdated data #[derive(Clone)] pub struct TableFullReplication { /// The membership manager of this node pub system: Arc, - /// Max number of faults allowed while replicating a record - pub max_faults: usize, } impl TableReplication for TableFullReplication { + type WriteSets = Vec>; + + fn storage_nodes(&self, _hash: &Hash) -> Vec { + let layout = self.system.cluster_layout(); + layout.current().all_nodes().to_vec() + } + fn read_nodes(&self, _hash: &Hash) -> Vec { vec![self.system.id] } @@ -26,26 +38,36 @@ impl TableReplication for TableFullReplication { 1 } - fn write_nodes(&self, _hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.layout.node_ids().to_vec() + fn write_sets(&self, hash: &Hash) -> Self::WriteSets { + vec![self.storage_nodes(hash)] } fn write_quorum(&self) -> usize { - let nmembers = self.system.ring.borrow().layout.node_ids().len(); - if nmembers > self.max_faults { - nmembers - self.max_faults + let nmembers = self.system.cluster_layout().current().all_nodes().len(); + + let max_faults = if nmembers > 1 { 1 } else { 0 }; + + if nmembers > max_faults { + nmembers - max_faults } else { 1 } } - fn max_write_errors(&self) -> usize { - self.max_faults - } fn partition_of(&self, _hash: &Hash) -> Partition { 0u16 } - fn partitions(&self) -> Vec<(Partition, Hash)> { - vec![(0u16, [0u8; 32].into())] + + fn sync_partitions(&self) -> SyncPartitions { + let layout = self.system.cluster_layout(); + let layout_version = layout.current().version; + SyncPartitions { + layout_version, + partitions: vec![SyncPartition { + partition: 0u16, + first_hash: [0u8; 32].into(), + last_hash: [0xff; 32].into(), + storage_sets: vec![layout.current().all_nodes().to_vec()], + }], + } } } diff --git a/src/table/replication/parameters.rs b/src/table/replication/parameters.rs index f00815a2..3649fad3 100644 --- a/src/table/replication/parameters.rs +++ b/src/table/replication/parameters.rs @@ -1,25 +1,43 @@ -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_util::data::*; /// Trait to describe how a table shall be replicated pub trait TableReplication: Send + Sync + 'static { + type WriteSets: AsRef>> + AsMut>> + Send + Sync + 'static; + // See examples in table_sharded.rs and table_fullcopy.rs // To understand various replication methods + /// The entire list of all nodes that store a partition + fn storage_nodes(&self, hash: &Hash) -> Vec; + /// Which nodes to send read requests to fn read_nodes(&self, hash: &Hash) -> Vec; - /// Responses needed to consider a read succesfull + /// Responses needed to consider a read successful fn read_quorum(&self) -> usize; /// Which nodes to send writes to - fn write_nodes(&self, hash: &Hash) -> Vec; - /// Responses needed to consider a write succesfull + fn write_sets(&self, hash: &Hash) -> Self::WriteSets; + /// Responses needed to consider a write successful in each set fn write_quorum(&self) -> usize; - fn max_write_errors(&self) -> usize; // Accessing partitions, for Merkle tree & sync /// Get partition for data with given hash fn partition_of(&self, hash: &Hash) -> Partition; - /// List of existing partitions - fn partitions(&self) -> Vec<(Partition, Hash)>; + /// List of partitions and nodes to sync with in current layout + fn sync_partitions(&self) -> SyncPartitions; +} + +#[derive(Debug)] +pub struct SyncPartitions { + pub layout_version: u64, + pub partitions: Vec, +} + +#[derive(Debug)] +pub struct SyncPartition { + pub partition: Partition, + pub first_hash: Hash, + pub last_hash: Hash, + pub storage_sets: Vec>, } diff --git a/src/table/replication/sharded.rs b/src/table/replication/sharded.rs index 1cf964af..e0245949 100644 --- a/src/table/replication/sharded.rs +++ b/src/table/replication/sharded.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use garage_rpc::ring::*; +use garage_rpc::layout::*; use garage_rpc::system::System; use garage_util::data::*; @@ -25,29 +25,59 @@ pub struct TableShardedReplication { } impl TableReplication for TableShardedReplication { + type WriteSets = WriteLock>>; + + fn storage_nodes(&self, hash: &Hash) -> Vec { + self.system.cluster_layout().storage_nodes_of(hash) + } + fn read_nodes(&self, hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.get_nodes(hash, self.replication_factor) + self.system.cluster_layout().read_nodes_of(hash) } fn read_quorum(&self) -> usize { self.read_quorum } - fn write_nodes(&self, hash: &Hash) -> Vec { - let ring = self.system.ring.borrow(); - ring.get_nodes(hash, self.replication_factor) + fn write_sets(&self, hash: &Hash) -> Self::WriteSets { + self.system.layout_manager.write_sets_of(hash) } fn write_quorum(&self) -> usize { self.write_quorum } - fn max_write_errors(&self) -> usize { - self.replication_factor - self.write_quorum - } fn partition_of(&self, hash: &Hash) -> Partition { - self.system.ring.borrow().partition_of(hash) + self.system.cluster_layout().current().partition_of(hash) } - fn partitions(&self) -> Vec<(Partition, Hash)> { - self.system.ring.borrow().partitions() + + fn sync_partitions(&self) -> SyncPartitions { + let layout = self.system.cluster_layout(); + let layout_version = layout.ack_map_min(); + + let mut partitions = layout + .current() + .partitions() + .map(|(partition, first_hash)| { + let storage_sets = layout.storage_sets_of(&first_hash); + SyncPartition { + partition, + first_hash, + last_hash: [0u8; 32].into(), // filled in just after + storage_sets, + } + }) + .collect::>(); + + for i in 0..partitions.len() { + partitions[i].last_hash = if i + 1 < partitions.len() { + partitions[i + 1].first_hash + } else { + [0xFFu8; 32].into() + }; + } + + SyncPartitions { + layout_version, + partitions, + } } } diff --git a/src/table/schema.rs b/src/table/schema.rs index 5cbf6c95..fc1a465e 100644 --- a/src/table/schema.rs +++ b/src/table/schema.rs @@ -6,6 +6,8 @@ use garage_util::migrate::Migrate; use crate::crdt::Crdt; +// =================================== PARTITION KEYS + /// Trait for field used to partition data pub trait PartitionKey: Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static @@ -29,6 +31,8 @@ impl PartitionKey for FixedBytes32 { } } +// =================================== SORT KEYS + /// Trait for field used to sort data pub trait SortKey: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static { /// Get the key used to sort @@ -47,6 +51,8 @@ impl SortKey for FixedBytes32 { } } +// =================================== SCHEMA + /// Trait for an entry in a table. It must be sortable and partitionnable. pub trait Entry: Crdt + PartialEq + Clone + Migrate + Send + Sync + 'static diff --git a/src/table/sync.rs b/src/table/sync.rs index 92a353c6..2d43b9fc 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -6,18 +6,19 @@ use arc_swap::ArcSwapOption; use async_trait::async_trait; use futures_util::stream::*; use opentelemetry::KeyValue; -use rand::Rng; +use rand::prelude::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use tokio::select; -use tokio::sync::{mpsc, watch}; +use tokio::sync::{mpsc, watch, Notify}; use garage_util::background::*; use garage_util::data::*; use garage_util::encode::{debug_serialize, nonversioned_encode}; use garage_util::error::{Error, OkOrMessage}; -use garage_rpc::ring::*; +use garage_rpc::layout::*; +use garage_rpc::rpc_helper::QuorumSetResultTracker; use garage_rpc::system::System; use garage_rpc::*; @@ -52,16 +53,6 @@ impl Rpc for SyncRpc { type Response = Result; } -#[derive(Debug, Clone)] -struct TodoPartition { - partition: Partition, - begin: Hash, - end: Hash, - - // Are we a node that stores this partition or not? - retain: bool, -} - impl TableSyncer { pub(crate) fn new( system: Arc, @@ -91,10 +82,10 @@ impl TableSyncer { bg.spawn_worker(SyncWorker { syncer: self.clone(), - ring_recv: self.system.ring.clone(), - ring: self.system.ring.borrow().clone(), + layout_notify: self.system.layout_notify(), + layout_digest: self.system.cluster_layout().sync_digest(), add_full_sync_rx, - todo: vec![], + todo: None, next_full_sync: Instant::now() + Duration::from_secs(20), }); } @@ -112,53 +103,56 @@ impl TableSyncer { async fn sync_partition( self: &Arc, - partition: &TodoPartition, + partition: &SyncPartition, must_exit: &mut watch::Receiver, ) -> Result<(), Error> { - if partition.retain { - let my_id = self.system.id; - - let nodes = self - .data - .replication - .write_nodes(&partition.begin) - .into_iter() - .filter(|node| *node != my_id) - .collect::>(); + let my_id = self.system.id; + let retain = partition.storage_sets.iter().any(|x| x.contains(&my_id)); + if retain { debug!( "({}) Syncing {:?} with {:?}...", F::TABLE_NAME, partition, - nodes + partition.storage_sets ); - let mut sync_futures = nodes - .iter() + let mut result_tracker = QuorumSetResultTracker::new( + &partition.storage_sets, + self.data.replication.write_quorum(), + ); + + let mut sync_futures = result_tracker + .nodes + .keys() + .copied() .map(|node| { - self.clone() - .do_sync_with(partition.clone(), *node, must_exit.clone()) + let must_exit = must_exit.clone(); + async move { + if node == my_id { + (node, Ok(())) + } else { + (node, self.do_sync_with(partition, node, must_exit).await) + } + } }) .collect::>(); - let mut n_errors = 0; - while let Some(r) = sync_futures.next().await { - if let Err(e) = r { - n_errors += 1; - warn!("({}) Sync error: {}", F::TABLE_NAME, e); + while let Some((node, res)) = sync_futures.next().await { + if let Err(e) = &res { + warn!("({}) Sync error with {:?}: {}", F::TABLE_NAME, node, e); } + result_tracker.register_result(node, res); } - if n_errors > self.data.replication.max_write_errors() { - return Err(Error::Message(format!( - "Sync failed with too many nodes (should have been: {:?}).", - nodes - ))); + + if result_tracker.too_many_failures() { + Err(result_tracker.quorum_error()) + } else { + Ok(()) } } else { - self.offload_partition(&partition.begin, &partition.end, must_exit) - .await?; + self.offload_partition(&partition.first_hash, &partition.last_hash, must_exit) + .await } - - Ok(()) } // Offload partition: this partition is not something we are storing, @@ -188,12 +182,7 @@ impl TableSyncer { } if !items.is_empty() { - let nodes = self - .data - .replication - .write_nodes(begin) - .into_iter() - .collect::>(); + let nodes = self.data.replication.storage_nodes(begin); if nodes.contains(&self.system.id) { warn!( "({}) Interrupting offload as partitions seem to have changed", @@ -217,7 +206,7 @@ impl TableSyncer { end, counter ); - self.offload_items(&items, &nodes[..]).await?; + self.offload_items(&items, &nodes).await?; } else { break; } @@ -244,7 +233,7 @@ impl TableSyncer { } self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, nodes, @@ -284,8 +273,8 @@ impl TableSyncer { } async fn do_sync_with( - self: Arc, - partition: TodoPartition, + self: &Arc, + partition: &SyncPartition, who: Uuid, must_exit: watch::Receiver, ) -> Result<(), Error> { @@ -305,7 +294,7 @@ impl TableSyncer { // If so, do nothing. let root_resp = self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -327,7 +316,7 @@ impl TableSyncer { SyncRpc::RootCkDifferent(true) => VecDeque::from(vec![root_ck_key]), x => { return Err(Error::Message(format!( - "Invalid respone to RootCkHash RPC: {}", + "Invalid response to RootCkHash RPC: {}", debug_serialize(x) ))); } @@ -361,7 +350,7 @@ impl TableSyncer { // and compare it with local node let remote_node = match self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -373,7 +362,7 @@ impl TableSyncer { SyncRpc::Node(_, node) => node, x => { return Err(Error::Message(format!( - "Invalid respone to GetNode RPC: {}", + "Invalid response to GetNode RPC: {}", debug_serialize(x) ))); } @@ -437,7 +426,7 @@ impl TableSyncer { let rpc_resp = self .system - .rpc + .rpc_helper() .call( &self.endpoint, who, @@ -455,7 +444,6 @@ impl TableSyncer { // ======= SYNCHRONIZATION PROCEDURE -- RECEIVER SIDE ====== -#[async_trait] impl EndpointHandler for TableSyncer { async fn handle(self: &Arc, message: &SyncRpc, from: NodeID) -> Result { match message { @@ -492,75 +480,41 @@ impl EndpointHandler for TableSync struct SyncWorker { syncer: Arc>, - ring_recv: watch::Receiver>, - ring: Arc, + + layout_notify: Arc, + layout_digest: SyncLayoutDigest, + add_full_sync_rx: mpsc::UnboundedReceiver<()>, - todo: Vec, next_full_sync: Instant, + + todo: Option, } impl SyncWorker { - fn add_full_sync(&mut self) { - let system = &self.syncer.system; - let data = &self.syncer.data; - - let my_id = system.id; - - self.todo.clear(); - - let partitions = data.replication.partitions(); - - for i in 0..partitions.len() { - let begin = partitions[i].1; - - let end = if i + 1 < partitions.len() { - partitions[i + 1].1 - } else { - [0xFFu8; 32].into() - }; - - let nodes = data.replication.write_nodes(&begin); - - let retain = nodes.contains(&my_id); - if !retain { - // Check if we have some data to send, otherwise skip - match data.store.range(begin..end) { - Ok(mut iter) => { - if iter.next().is_none() { - continue; - } - } - Err(e) => { - warn!("DB error in add_full_sync: {}", e); - continue; - } - } - } - - self.todo.push(TodoPartition { - partition: partitions[i].0, - begin, - end, - retain, - }); + fn check_add_full_sync(&mut self) { + let layout_digest = self.syncer.system.cluster_layout().sync_digest(); + if layout_digest != self.layout_digest { + self.layout_digest = layout_digest; + info!( + "({}) Layout versions changed ({:?}), adding full sync to syncer todo list", + F::TABLE_NAME, + layout_digest, + ); + self.add_full_sync(); } - - self.next_full_sync = Instant::now() + ANTI_ENTROPY_INTERVAL; } - fn pop_task(&mut self) -> Option { - if self.todo.is_empty() { - return None; - } + fn add_full_sync(&mut self) { + let mut partitions = self.syncer.data.replication.sync_partitions(); + info!( + "{}: Adding full sync for ack layout version {}", + F::TABLE_NAME, + partitions.layout_version + ); - let i = rand::thread_rng().gen_range(0..self.todo.len()); - if i == self.todo.len() - 1 { - self.todo.pop() - } else { - let replacement = self.todo.pop().unwrap(); - let ret = std::mem::replace(&mut self.todo[i], replacement); - Some(ret) - } + partitions.partitions.shuffle(&mut thread_rng()); + self.todo = Some(partitions); + self.next_full_sync = Instant::now() + ANTI_ENTROPY_INTERVAL; } } @@ -572,14 +526,48 @@ impl Worker for SyncWorker { fn status(&self) -> WorkerStatus { WorkerStatus { - queue_length: Some(self.todo.len() as u64), + queue_length: Some(self.todo.as_ref().map(|x| x.partitions.len()).unwrap_or(0) as u64), ..Default::default() } } async fn work(&mut self, must_exit: &mut watch::Receiver) -> Result { - if let Some(partition) = self.pop_task() { - self.syncer.sync_partition(&partition, must_exit).await?; + self.check_add_full_sync(); + + if let Some(todo) = &mut self.todo { + let partition = todo.partitions.pop().unwrap(); + + // process partition + if let Err(e) = self.syncer.sync_partition(&partition, must_exit).await { + error!( + "{}: Failed to sync partition {:?}: {}", + F::TABLE_NAME, + partition, + e + ); + // if error, put partition back at the other side of the queue, + // so that other partitions will be tried in the meantime + todo.partitions.insert(0, partition); + // TODO: returning an error here will cause the background job worker + // to delay this task for some time, but maybe we don't want to + // delay it if there are lots of failures from nodes that are gone + // (we also don't want zero delays as that will cause lots of useless retries) + return Err(e); + } + + if todo.partitions.is_empty() { + info!( + "{}: Completed full sync for ack layout version {}", + F::TABLE_NAME, + todo.layout_version + ); + self.syncer + .system + .layout_manager + .sync_table_until(F::TABLE_NAME, todo.layout_version); + self.todo = None; + } + Ok(WorkerState::Busy) } else { Ok(WorkerState::Idle) @@ -593,22 +581,16 @@ impl Worker for SyncWorker { self.add_full_sync(); } }, - _ = self.ring_recv.changed() => { - let new_ring = self.ring_recv.borrow(); - if !Arc::ptr_eq(&new_ring, &self.ring) { - self.ring = new_ring.clone(); - drop(new_ring); - debug!("({}) Ring changed, adding full sync to syncer todo list", F::TABLE_NAME); - self.add_full_sync(); - } + _ = self.layout_notify.notified() => { + self.check_add_full_sync(); }, _ = tokio::time::sleep_until(self.next_full_sync.into()) => { self.add_full_sync(); } } - match self.todo.is_empty() { - false => WorkerState::Busy, - true => WorkerState::Idle, + match self.todo.is_some() { + true => WorkerState::Busy, + false => WorkerState::Idle, } } } diff --git a/src/table/table.rs b/src/table/table.rs index 7ad79677..c96f4731 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -2,7 +2,6 @@ use std::borrow::Borrow; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; -use async_trait::async_trait; use futures::stream::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -20,6 +19,7 @@ use garage_util::error::Error; use garage_util::metrics::RecordDuration; use garage_util::migrate::Migrate; +use garage_rpc::rpc_helper::QuorumSetResultTracker; use garage_rpc::system::System; use garage_rpc::*; @@ -80,6 +80,8 @@ impl Table { let syncer = TableSyncer::new(system.clone(), data.clone(), merkle_updater.clone()); let gc = TableGc::new(system.clone(), data.clone()); + system.layout_manager.add_table(F::TABLE_NAME); + let table = Arc::new(Self { system, data, @@ -117,16 +119,16 @@ impl Table { async fn insert_internal(&self, e: &F::E) -> Result<(), Error> { let hash = e.partition_key().hash(); - let who = self.data.replication.write_nodes(&hash); + let who = self.data.replication.write_sets(&hash); let e_enc = Arc::new(ByteBuf::from(e.encode()?)); let rpc = TableRpc::::Update(vec![e_enc]); self.system - .rpc - .try_call_many( + .rpc_helper() + .try_write_many_sets( &self.endpoint, - &who[..], + who.as_ref(), rpc, RequestStrategy::with_priority(PRIO_NORMAL) .with_quorum(self.data.replication.write_quorum()), @@ -141,7 +143,7 @@ impl Table { self.data.queue_insert(tx, e) } - pub async fn insert_many(&self, entries: I) -> Result<(), Error> + pub async fn insert_many(self: &Arc, entries: I) -> Result<(), Error> where I: IntoIterator + Send + Sync, IE: Borrow + Send + Sync, @@ -159,51 +161,127 @@ impl Table { Ok(()) } - async fn insert_many_internal(&self, entries: I) -> Result<(), Error> + async fn insert_many_internal(self: &Arc, entries: I) -> Result<(), Error> where I: IntoIterator + Send + Sync, IE: Borrow + Send + Sync, { - let mut call_list: HashMap<_, Vec<_>> = HashMap::new(); + // The different items will have to be stored on possibly different nodes. + // We will here batch all items into a single request for each concerned + // node, with all of the entries it must store within that request. + // Each entry has to be saved to a specific list of "write sets", i.e. a set + // of node within which a quorum must be achieved. In normal operation, there + // is a single write set which corresponds to the quorum in the current + // cluster layout, but when the layout is updated, multiple write sets might + // have to be handled at once. Here, since we are sending many entries, we + // will have to handle many write sets in all cases. The algorithm is thus + // to send one request to each node with all the items it must save, + // and keep track of the OK responses within each write set: if for all sets + // a quorum of nodes has answered OK, then the insert has succeeded and + // consistency properties (read-after-write) are preserved. + let quorum = self.data.replication.write_quorum(); + + // Serialize all entries and compute the write sets for each of them. + // In the case of sharded table replication, this also takes an "ack lock" + // to the layout manager to avoid ack'ing newer versions which are not + // taken into account by writes in progress (the ack can happen later, once + // all writes that didn't take the new layout into account are finished). + // These locks are released when entries_vec is dropped, i.e. when this + // function returns. + let mut entries_vec = Vec::new(); for entry in entries.into_iter() { let entry = entry.borrow(); let hash = entry.partition_key().hash(); - let who = self.data.replication.write_nodes(&hash); + let mut write_sets = self.data.replication.write_sets(&hash); + for set in write_sets.as_mut().iter_mut() { + // Sort nodes in each write sets to merge write sets with same + // nodes but in possibly different orders + set.sort(); + } let e_enc = Arc::new(ByteBuf::from(entry.encode()?)); - for node in who { - call_list.entry(node).or_default().push(e_enc.clone()); + entries_vec.push((write_sets, e_enc)); + } + + if entries_vec.is_empty() { + return Ok(()); + } + + // Compute a deduplicated list of all of the write sets, + // and compute an index from each node to the position of the sets in which + // it takes part, to optimize the detection of a quorum. + let mut write_sets = entries_vec + .iter() + .flat_map(|(wss, _)| wss.as_ref().iter().map(|ws| ws.as_slice())) + .collect::>(); + write_sets.sort(); + write_sets.dedup(); + + let mut result_tracker = QuorumSetResultTracker::new(&write_sets, quorum); + + // Build a map of all nodes to the entries that must be sent to that node. + let mut call_list: HashMap> = HashMap::new(); + for (write_sets, entry_enc) in entries_vec.iter() { + for write_set in write_sets.as_ref().iter() { + for node in write_set.iter() { + let node_entries = call_list.entry(*node).or_default(); + match node_entries.last() { + Some(x) if Arc::ptr_eq(x, entry_enc) => { + // skip if entry already in list to send to this node + // (could happen if node is in several write sets for this entry) + } + _ => { + node_entries.push(entry_enc.clone()); + } + } + } } } - let call_futures = call_list.drain().map(|(node, entries)| async move { - let rpc = TableRpc::::Update(entries); - - let resp = self - .system - .rpc - .call( - &self.endpoint, - node, - rpc, - RequestStrategy::with_priority(PRIO_NORMAL), - ) - .await?; - Ok::<_, Error>((node, resp)) + // Build futures to actually perform each of the corresponding RPC calls + let call_futures = call_list.into_iter().map(|(node, entries)| { + let this = self.clone(); + async move { + let rpc = TableRpc::::Update(entries); + let resp = this + .system + .rpc_helper() + .call( + &this.endpoint, + node, + rpc, + RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(quorum), + ) + .await; + (node, resp) + } }); - let mut resps = call_futures.collect::>(); - let mut errors = vec![]; - while let Some(resp) = resps.next().await { - if let Err(e) = resp { - errors.push(e); + // Run all requests in parallel thanks to FuturesUnordered, and collect results. + let mut resps = call_futures.collect::>(); + + while let Some((node, resp)) = resps.next().await { + result_tracker.register_result(node, resp.map(|_| ())); + + if result_tracker.all_quorums_ok() { + // Success + + // Continue all other requests in background + tokio::spawn(async move { + resps.collect::)>>().await; + }); + + return Ok(()); + } + + if result_tracker.too_many_failures() { + // Too many errors in this set, we know we won't get a quorum + break; } } - if errors.len() > self.data.replication.max_write_errors() { - Err(Error::Message("Too many errors".into())) - } else { - Ok(()) - } + + // Failure, could not get quorum within at least one set + Err(result_tracker.quorum_error()) } pub async fn get( @@ -236,14 +314,13 @@ impl Table { let rpc = TableRpc::::ReadEntry(partition_key.clone(), sort_key.clone()); let resps = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, rpc, RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(self.data.replication.read_quorum()) - .interrupt_after_quorum(true), + .with_quorum(self.data.replication.read_quorum()), ) .await?; @@ -332,14 +409,13 @@ impl Table { let resps = self .system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, - &who[..], + &who, rpc, RequestStrategy::with_priority(PRIO_NORMAL) - .with_quorum(self.data.replication.read_quorum()) - .interrupt_after_quorum(true), + .with_quorum(self.data.replication.read_quorum()), ) .await?; @@ -411,7 +487,7 @@ impl Table { async fn repair_on_read(&self, who: &[Uuid], what: F::E) -> Result<(), Error> { let what_enc = Arc::new(ByteBuf::from(what.encode()?)); self.system - .rpc + .rpc_helper() .try_call_many( &self.endpoint, who, @@ -423,7 +499,6 @@ impl Table { } } -#[async_trait] impl EndpointHandler> for Table { async fn handle( self: &Arc, diff --git a/src/table/util.rs b/src/table/util.rs index 0b10cf3f..663a7e11 100644 --- a/src/table/util.rs +++ b/src/table/util.rs @@ -34,8 +34,9 @@ impl DeletedFilter { } } -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub enum EnumerationOrder { + #[default] Forward, Reverse, } @@ -49,9 +50,3 @@ impl EnumerationOrder { } } } - -impl Default for EnumerationOrder { - fn default() -> Self { - EnumerationOrder::Forward - } -} diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 2e6231f6..46fa6590 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,44 +14,41 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.2", path = "../db" } +garage_db.workspace = true +garage_net.workspace = true -arc-swap = "1.0" -async-trait = "0.1" -blake2 = "0.10" -bytes = "1.0" -digest = "0.10" -err-derive = "0.3" -git-version = "0.3.4" -hexdump = "0.1" -xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } -hex = "0.4" -lazy_static = "1.4" -tracing = "0.1" -rand = "0.8" -sha2 = "0.10" +arc-swap.workspace = true +async-trait.workspace = true +blake2.workspace = true +bytesize.workspace = true +thiserror.workspace = true +hexdump.workspace = true +xxhash-rust.workspace = true +hex.workspace = true +lazy_static.workspace = true +tracing.workspace = true +rand.workspace = true +sha2.workspace = true -chrono = "0.4" -rmp-serde = "0.15" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_json = "1.0" -toml = "0.6" +chrono.workspace = true +rmp-serde.workspace = true +serde.workspace = true +serde_json.workspace = true +toml.workspace = true -futures = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +futures.workspace = true +tokio.workspace = true -netapp = "0.5" +http.workspace = true +hyper.workspace = true -http = "0.2" -hyper = "0.14" - -opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +opentelemetry.workspace = true [build-dependencies] -rustc_version = "0.4.0" +rustc_version.workspace = true [dev-dependencies] -mktemp = "0.5" +mktemp.workspace = true [features] k2v = [] diff --git a/src/util/async_hash.rs b/src/util/async_hash.rs deleted file mode 100644 index 5631ea6b..00000000 --- a/src/util/async_hash.rs +++ /dev/null @@ -1,61 +0,0 @@ -use bytes::Bytes; -use digest::Digest; - -use tokio::sync::mpsc; -use tokio::task::JoinHandle; - -use crate::data::*; - -/// Compute the sha256 of a slice, -/// spawning on a tokio thread for CPU-intensive processing -/// The argument has to be an owned Bytes, as it is moved out to a new thread. -pub async fn async_sha256sum(data: Bytes) -> Hash { - tokio::task::spawn_blocking(move || sha256sum(&data)) - .await - .unwrap() -} - -/// Compute the blake2sum of a slice, -/// spawning on a tokio thread for CPU-intensive processing. -/// The argument has to be an owned Bytes, as it is moved out to a new thread. -pub async fn async_blake2sum(data: Bytes) -> Hash { - tokio::task::spawn_blocking(move || blake2sum(&data)) - .await - .unwrap() -} - -// ---- - -pub struct AsyncHasher { - sendblk: mpsc::Sender, - task: JoinHandle>, -} - -impl AsyncHasher { - pub fn new() -> Self { - let (sendblk, mut recvblk) = mpsc::channel::(1); - let task = tokio::task::spawn_blocking(move || { - let mut digest = D::new(); - while let Some(blk) = recvblk.blocking_recv() { - digest.update(&blk[..]); - } - digest.finalize() - }); - Self { sendblk, task } - } - - pub async fn update(&self, b: Bytes) { - self.sendblk.send(b).await.unwrap(); - } - - pub async fn finalize(self) -> digest::Output { - drop(self.sendblk); - self.task.await.unwrap() - } -} - -impl Default for AsyncHasher { - fn default() -> Self { - Self::new() - } -} diff --git a/src/util/background/worker.rs b/src/util/background/worker.rs index 8165e2cb..3c938b7e 100644 --- a/src/util/background/worker.rs +++ b/src/util/background/worker.rs @@ -14,7 +14,7 @@ use crate::background::{WorkerInfo, WorkerStatus}; use crate::error::Error; use crate::time::now_msec; -// All workers that haven't exited for this time after an exit signal was recieved +// All workers that haven't exited for this time after an exit signal was received // will be interrupted in the middle of whatever they are doing. const EXIT_DEADLINE: Duration = Duration::from_secs(8); @@ -54,7 +54,7 @@ pub trait Worker: Send { async fn work(&mut self, must_exit: &mut watch::Receiver) -> Result; /// Wait for work: await for some task to become available. This future can be interrupted in - /// the middle for any reason, for example if an interrupt signal was recieved. + /// the middle for any reason, for example if an interrupt signal was received. async fn wait_for_work(&mut self) -> WorkerState; } @@ -115,32 +115,39 @@ impl WorkerProcessor { trace!("{} (TID {}): {:?}", worker.worker.name(), worker.task_id, worker.state); // Save worker info - let mut wi = self.worker_info.lock().unwrap(); - match wi.get_mut(&worker.task_id) { - Some(i) => { - i.state = worker.state; - i.status = worker.worker.status(); - i.errors = worker.errors; - i.consecutive_errors = worker.consecutive_errors; - if worker.last_error.is_some() { - i.last_error = worker.last_error.take(); + { + let mut wi = self.worker_info.lock().unwrap(); + match wi.get_mut(&worker.task_id) { + Some(i) => { + i.state = worker.state; + i.status = worker.worker.status(); + i.errors = worker.errors; + i.consecutive_errors = worker.consecutive_errors; + if worker.last_error.is_some() { + i.last_error = worker.last_error.take(); + } + } + None => { + wi.insert(worker.task_id, WorkerInfo { + name: worker.worker.name(), + state: worker.state, + status: worker.worker.status(), + errors: worker.errors, + consecutive_errors: worker.consecutive_errors, + last_error: worker.last_error.take(), + }); } - } - None => { - wi.insert(worker.task_id, WorkerInfo { - name: worker.worker.name(), - state: worker.state, - status: worker.worker.status(), - errors: worker.errors, - consecutive_errors: worker.consecutive_errors, - last_error: worker.last_error.take(), - }); } } if worker.state == WorkerState::Done { info!("Worker {} (TID {}) exited", worker.worker.name(), worker.task_id); } else { + // Yield to the Tokio scheduler between consecutive Busy steps so + // that a worker which never suspends on its own cannot starve other tasks. + if worker.state == WorkerState::Busy { + tokio::task::yield_now().await; + } workers.push(async move { worker.step().await; worker diff --git a/src/util/config.rs b/src/util/config.rs index 2176353e..eb889ebe 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -1,11 +1,12 @@ //! Contains type and functions related to Garage configuration file -use std::io::Read; +use std::convert::TryFrom; use std::net::SocketAddr; use std::path::PathBuf; use serde::{de, Deserialize}; use crate::error::Error; +use crate::socket_address::UnixOrTCPSocketAddress; /// Represent the whole configuration #[derive(Deserialize, Debug, Clone)] @@ -13,18 +14,56 @@ pub struct Config { /// Path where to store metadata. Should be fast, but low volume pub metadata_dir: PathBuf, /// Path where to store data. Can be slower, but need higher volume - pub data_dir: PathBuf, + pub data_dir: DataDirEnum, + + /// Whether to fsync after all metadata transactions (disabled by default) + #[serde(default)] + pub metadata_fsync: bool, + /// Whether to fsync after all data block writes (disabled by default) + #[serde(default)] + pub data_fsync: bool, + + /// Disable automatic scrubbing of the data directory + #[serde(default)] + pub disable_scrub: bool, + + /// Use local timezone + #[serde(default)] + pub use_local_tz: bool, + + /// Optional directory where metadata snapshots will be store + pub metadata_snapshots_dir: Option, + + /// Automatic snapshot interval for metadata + #[serde(default)] + pub metadata_auto_snapshot_interval: Option, /// Size of data blocks to save to disk - #[serde(default = "default_block_size")] + #[serde( + deserialize_with = "deserialize_capacity", + default = "default_block_size" + )] pub block_size: usize, - /// Replication mode. Supported values: - /// - none, 1 -> no replication - /// - 2 -> 2-way replication - /// - 3 -> 3-way replication - // (we can add more aliases for this later) - pub replication_mode: String, + /// Maximum number of parallel block writes per PUT request + /// Higher values improve throughput but increase memory usage + /// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #[serde(default = "default_block_max_concurrent_writes_per_request")] + pub block_max_concurrent_writes_per_request: usize, + /// Number of replicas. Can be any positive integer, but uneven numbers are more favorable. + /// - 1 for single-node clusters, or to disable replication + /// - 3 is the recommended and supported setting. + #[serde(default)] + pub replication_factor: Option, + + /// Consistency mode for all for requests through this node + /// - Degraded -> Disable read quorum + /// - Dangerous -> Disable read and write quorum + #[serde(default = "default_consistency_mode")] + pub consistency_mode: String, + + /// Legacy option + pub replication_mode: Option, /// Zstd compression level used on data blocks #[serde( @@ -33,22 +72,45 @@ pub struct Config { )] pub compression_level: Option, + /// Maximum amount of block data to buffer in RAM for sending to + /// remote nodes when these nodes are on slower links + #[serde( + deserialize_with = "deserialize_capacity", + default = "default_block_ram_buffer_max" + )] + pub block_ram_buffer_max: usize, + + /// Maximum number of concurrent reads of block files on disk + #[serde(default = "default_block_max_concurrent_reads")] + pub block_max_concurrent_reads: usize, + + /// Skip the permission check of secret files. Useful when + /// POSIX ACLs (or more complex chmods) are used. + #[serde(default)] + pub allow_world_readable_secrets: bool, + /// RPC secret key: 32 bytes hex encoded pub rpc_secret: Option, /// Optional file where RPC secret key is read from - pub rpc_secret_file: Option, - + pub rpc_secret_file: Option, /// Address to bind for RPC pub rpc_bind_addr: SocketAddr, + /// Bind outgoing sockets to rpc_bind_addr's IP address as well + #[serde(default)] + pub rpc_bind_outgoing: bool, /// Public IP address of this node pub rpc_public_addr: Option, - /// Timeout for Netapp's ping messagess + /// In case `rpc_public_addr` was not set, this can filter + /// the addresses announced to other peers to a specific subnet. + pub rpc_public_addr_subnet: Option, + + /// Timeout for Netapp's ping messages pub rpc_ping_timeout_msec: Option, /// Timeout for Netapp RPC calls pub rpc_timeout_msec: Option, - // -- Bootstraping and discovery + // -- Bootstrapping and discovery /// Bootstrap peers RPC address #[serde(default)] pub bootstrap_peers: Vec, @@ -61,16 +123,17 @@ pub struct Config { pub kubernetes_discovery: Option, // -- DB - /// Database engine to use for metadata (options: sled, sqlite, lmdb) + /// Database engine to use for metadata (options: sqlite, lmdb) #[serde(default = "default_db_engine")] pub db_engine: String, - /// Sled cache size, in bytes - #[serde(default = "default_sled_cache_capacity")] - pub sled_cache_capacity: u64, - /// Sled flush interval in milliseconds - #[serde(default = "default_sled_flush_every_ms")] - pub sled_flush_every_ms: u64, + /// LMDB map size + #[serde(deserialize_with = "deserialize_capacity", default)] + pub lmdb_map_size: usize, + + /// Fjall block cache size + #[serde(deserialize_with = "deserialize_capacity", default)] + pub fjall_block_cache_size: usize, // -- APIs /// Configuration for S3 api @@ -85,13 +148,37 @@ pub struct Config { /// Configuration for the admin API endpoint #[serde(default = "Default::default")] pub admin: AdminConfig, + + /// Allow punycode in bucket names + #[serde(default)] + pub allow_punycode: bool, +} + +/// Value for data_dir: either a single directory or a list of dirs with attributes +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum DataDirEnum { + Single(PathBuf), + Multiple(Vec), +} + +#[derive(Deserialize, Debug, Clone)] +pub struct DataDir { + /// Path to the data directory + pub path: PathBuf, + /// Capacity of the drive (required if read_only is false) + #[serde(default)] + pub capacity: Option, + /// Whether this is a legacy read-only path (capacity should be None) + #[serde(default)] + pub read_only: bool, } /// Configuration for S3 api #[derive(Deserialize, Debug, Clone)] pub struct S3ApiConfig { /// Address and port to bind for api serving - pub api_bind_addr: Option, + pub api_bind_addr: Option, /// S3 region to use pub s3_region: String, /// Suffix to remove from domain name to find bucket. If None, @@ -103,40 +190,54 @@ pub struct S3ApiConfig { #[derive(Deserialize, Debug, Clone)] pub struct K2VApiConfig { /// Address and port to bind for api serving - pub api_bind_addr: SocketAddr, + pub api_bind_addr: UnixOrTCPSocketAddress, } /// Configuration for serving files as normal web server #[derive(Deserialize, Debug, Clone)] pub struct WebConfig { /// Address and port to bind for web serving - pub bind_addr: SocketAddr, + pub bind_addr: UnixOrTCPSocketAddress, /// Suffix to remove from domain name to find bucket pub root_domain: String, + /// Whether to add the requested domain to exported Prometheus metrics + #[serde(default)] + pub add_host_to_metrics: bool, } /// Configuration for the admin and monitoring HTTP API #[derive(Deserialize, Debug, Clone, Default)] pub struct AdminConfig { /// Address and port to bind for admin API serving - pub api_bind_addr: Option, + pub api_bind_addr: Option, /// Bearer token to use to scrape metrics pub metrics_token: Option, /// File to read metrics token from - pub metrics_token_file: Option, + pub metrics_token_file: Option, /// Bearer token to use to access Admin API endpoints pub admin_token: Option, /// File to read admin token from - pub admin_token_file: Option, + pub admin_token_file: Option, /// OTLP server to where to export traces pub trace_sink: Option, } +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "lowercase")] +pub enum ConsulDiscoveryAPI { + #[default] + Catalog, + Agent, +} + #[derive(Deserialize, Debug, Clone)] pub struct ConsulDiscoveryConfig { + /// The consul api to use when registering: either `catalog` (the default) or `agent` + #[serde(default)] + pub api: ConsulDiscoveryAPI, /// Consul http or https address to connect to to discover more peers pub consul_http_addr: String, /// Consul service name to use @@ -147,9 +248,17 @@ pub struct ConsulDiscoveryConfig { pub client_cert: Option, /// Client TLS key to use when connecting to Consul pub client_key: Option, + /// /// Token to use for connecting to consul + pub token: Option, /// Skip TLS hostname verification #[serde(default)] pub tls_skip_verify: bool, + /// Additional tags to add to the service + #[serde(default)] + pub tags: Vec, + /// Additional service metadata to add + #[serde(default)] + pub meta: Option>, } #[derive(Deserialize, Debug, Clone)] @@ -163,80 +272,32 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } -fn default_db_engine() -> String { - "sled".into() +pub fn default_block_max_concurrent_writes_per_request() -> usize { + 3 +} +/// Read and parse configuration +pub fn read_config(config_file: PathBuf) -> Result { + let config = std::fs::read_to_string(config_file)?; + + Ok(toml::from_str(&config)?) } -fn default_sled_cache_capacity() -> u64 { - 128 * 1024 * 1024 -} -fn default_sled_flush_every_ms() -> u64 { - 2000 +fn default_db_engine() -> String { + "lmdb".into() } + fn default_block_size() -> usize { 1048576 } - -/// Read and parse configuration -pub fn read_config(config_file: PathBuf) -> Result { - let mut file = std::fs::OpenOptions::new() - .read(true) - .open(config_file.as_path())?; - - let mut config = String::new(); - file.read_to_string(&mut config)?; - - let mut parsed_config: Config = toml::from_str(&config)?; - - secret_from_file( - &mut parsed_config.rpc_secret, - &parsed_config.rpc_secret_file, - "rpc_secret", - )?; - secret_from_file( - &mut parsed_config.admin.metrics_token, - &parsed_config.admin.metrics_token_file, - "admin.metrics_token", - )?; - secret_from_file( - &mut parsed_config.admin.admin_token, - &parsed_config.admin.admin_token_file, - "admin.admin_token", - )?; - - Ok(parsed_config) +fn default_block_ram_buffer_max() -> usize { + 256 * 1024 * 1024 +} +fn default_block_max_concurrent_reads() -> usize { + 16 } -fn secret_from_file( - secret: &mut Option, - secret_file: &Option, - name: &'static str, -) -> Result<(), Error> { - match (&secret, &secret_file) { - (_, None) => { - // no-op - } - (Some(_), Some(_)) => { - return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); - } - (None, Some(file_path)) => { - #[cfg(unix)] - if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") { - use std::os::unix::fs::MetadataExt; - let metadata = std::fs::metadata(&file_path)?; - if metadata.mode() & 0o077 != 0 { - return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); - } - } - let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?; - let mut secret_buf = String::new(); - file.read_to_string(&mut secret_buf)?; - // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. - // also editors sometimes add a trailing newline - *secret = Some(String::from(secret_buf.trim_end())); - } - } - Ok(()) +fn default_consistency_mode() -> String { + "consistent".into() } fn default_compression() -> Option { @@ -247,8 +308,6 @@ fn deserialize_compression<'de, D>(deserializer: D) -> Result, D::Er where D: de::Deserializer<'de>, { - use std::convert::TryFrom; - struct OptionVisitor; impl<'de> serde::de::Visitor<'de> for OptionVisitor { @@ -293,6 +352,50 @@ where deserializer.deserialize_any(OptionVisitor) } +fn deserialize_capacity<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + struct CapacityVisitor; + + impl<'de> serde::de::Visitor<'de> for CapacityVisitor { + type Value = usize; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("int or ''") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + value + .parse::() + .map(|x| x.as_u64()) + .map_err(|e| E::custom(format!("invalid capacity value: {}", e))) + .and_then(|v| { + usize::try_from(v) + .map_err(|_| E::custom("capacity value out of bound".to_owned())) + }) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned())) + } + + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned())) + } + } + + deserializer.deserialize_any(CapacityVisitor) +} + #[cfg(test)] mod tests { use crate::error::Error; @@ -308,7 +411,7 @@ mod tests { r#" metadata_dir = "/tmp/garage/meta" data_dir = "/tmp/garage/data" - replication_mode = "3" + replication_factor = 3 rpc_bind_addr = "[::]:3901" rpc_secret = "foo" @@ -325,83 +428,4 @@ mod tests { Ok(()) } - - #[test] - fn test_rpc_secret_file_works() -> Result<(), Error> { - let path_secret = mktemp::Temp::new_file()?; - let mut file_secret = File::create(path_secret.as_path())?; - writeln!(file_secret, "foo")?; - drop(file_secret); - - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - let path_secret_path = path_secret.as_path(); - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret_file = "{}" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "#, - path_secret_path.display() - )?; - let config = super::read_config(path_config.to_path_buf())?; - assert_eq!("foo", config.rpc_secret.unwrap()); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let metadata = std::fs::metadata(&path_secret_path)?; - let mut perm = metadata.permissions(); - perm.set_mode(0o660); - std::fs::set_permissions(&path_secret_path, perm)?; - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false"); - assert!(super::read_config(path_config.to_path_buf()).is_err()); - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true"); - assert!(super::read_config(path_config.to_path_buf()).is_ok()); - } - - drop(path_config); - drop(path_secret); - drop(file_config); - Ok(()) - } - - #[test] - fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret= "dummy" - rpc_secret_file = "dummy" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "# - )?; - assert_eq!( - "only one of `rpc_secret` and `rpc_secret_file` can be set", - super::read_config(path_config.to_path_buf()) - .unwrap_err() - .to_string() - ); - drop(path_config); - drop(file_config); - Ok(()) - } } diff --git a/src/util/crdt/crdt.rs b/src/util/crdt/crdt.rs index 06876897..fdf63084 100644 --- a/src/util/crdt/crdt.rs +++ b/src/util/crdt/crdt.rs @@ -33,8 +33,8 @@ pub trait Crdt { /// arises very often, for example with a Lww or a LwwMap: the value type has to be a CRDT so that /// we have a rule for what to do when timestamps aren't enough to disambiguate (in a distributed /// system, anything can happen!), and with AutoCrdt the rule is to make an arbitrary (but -/// determinstic) choice between the two. When using an Option instead with this impl, ambiguity -/// cases are explicitely stored as None, which allows us to detect the ambiguity and handle it in +/// deterministic) choice between the two. When using an Option instead with this impl, ambiguity +/// cases are explicitly stored as None, which allows us to detect the ambiguity and handle it in /// the way we want. (this can only work if we are happy with losing the value when an ambiguity /// arises) impl Crdt for Option diff --git a/src/util/crdt/lww.rs b/src/util/crdt/lww.rs index 958844c9..80747406 100644 --- a/src/util/crdt/lww.rs +++ b/src/util/crdt/lww.rs @@ -16,7 +16,7 @@ use crate::crdt::crdt::*; /// In our case, we add the constraint that the value that is wrapped inside the LWW CRDT must /// itself be a CRDT: in the case when the timestamp does not allow us to decide on which value to /// keep, the merge rule of the inner CRDT is applied on the wrapped values. (Note that all types -/// that implement the `Ord` trait get a default CRDT implemetnation that keeps the maximum value. +/// that implement the `Ord` trait get a default CRDT implementation that keeps the maximum value. /// This enables us to use LWW directly with primitive data types such as numbers or strings. It is /// generally desirable in this case to never explicitly produce LWW values with the same timestamp /// but different inner values, as the rule to keep the maximum value isn't generally the desired @@ -28,9 +28,9 @@ use crate::crdt::crdt::*; /// /// Given that clocks are not too desynchronized, this assumption /// is enough for most cases, as there is few chance that two humans -/// coordonate themself faster than the time difference between two NTP servers. +/// coordinate themself faster than the time difference between two NTP servers. /// -/// As a more concret example, let's suppose you want to upload a file +/// As a more concrete example, let's suppose you want to upload a file /// with the same key (path) in the same bucket at the very same time. /// For each request, the file will be timestamped by the receiving server /// and may differ from what you observed with your atomic clock! @@ -84,16 +84,16 @@ where &self.v } - /// Take the value inside the CRDT (discards the timesamp) + /// Take the value inside the CRDT (discards the timestamp) pub fn take(self) -> T { self.v } /// Get a mutable reference to the CRDT's value /// - /// This is usefull to mutate the inside value without changing the LWW timestamp. + /// This is useful to mutate the inside value without changing the LWW timestamp. /// When such mutation is done, the merge between two LWW values is done using the inner - /// CRDT's merge operation. This is usefull in the case where the inner CRDT is a large + /// CRDT's merge operation. This is useful in the case where the inner CRDT is a large /// data type, such as a map, and we only want to change a single item in the map. /// To do this, we can produce a "CRDT delta", i.e. a LWW that contains only the modification. /// This delta consists in a LWW with the same timestamp, and the map diff --git a/src/util/crdt/lww_map.rs b/src/util/crdt/lww_map.rs index 88113856..def0ebeb 100644 --- a/src/util/crdt/lww_map.rs +++ b/src/util/crdt/lww_map.rs @@ -109,7 +109,7 @@ where } /// Takes all of the values of the map and returns them. The current map is reset to the - /// empty map. This is very usefull to produce in-place a new map that contains only a delta + /// empty map. This is very useful to produce in-place a new map that contains only a delta /// that modifies a certain value: /// /// ```ignore @@ -162,7 +162,7 @@ where } } - /// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values. + /// Gets a reference to all of the items, as a slice. Useful to iterate on all map values. /// In most case you will want to ignore the timestamp (second item of the tuple). pub fn items(&self) -> &[(K, u64, V)] { &self.vals[..] diff --git a/src/util/crdt/map.rs b/src/util/crdt/map.rs index 5d1e1520..adac3c38 100644 --- a/src/util/crdt/map.rs +++ b/src/util/crdt/map.rs @@ -57,7 +57,7 @@ where Err(_) => None, } } - /// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values. + /// Gets a reference to all of the items, as a slice. Useful to iterate on all map values. pub fn items(&self) -> &[(K, V)] { &self.vals[..] } diff --git a/src/util/data.rs b/src/util/data.rs index bdd8daee..1fe7dfe0 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -83,17 +83,30 @@ impl FixedBytes32 { ret.copy_from_slice(by); Some(Self(ret)) } + /// Return the next hash + pub fn increment(&self) -> Option { + let mut ret = *self; + for byte in ret.0.iter_mut().rev() { + if *byte == u8::MAX { + *byte = 0; + } else { + *byte = *byte + 1; + return Some(ret); + } + } + return None; + } } -impl From for FixedBytes32 { - fn from(node_id: netapp::NodeID) -> FixedBytes32 { +impl From for FixedBytes32 { + fn from(node_id: garage_net::NodeID) -> FixedBytes32 { FixedBytes32::try_from(node_id.as_ref()).unwrap() } } -impl From for netapp::NodeID { - fn from(bytes: FixedBytes32) -> netapp::NodeID { - netapp::NodeID::from_slice(bytes.as_slice()).unwrap() +impl From for garage_net::NodeID { + fn from(bytes: FixedBytes32) -> garage_net::NodeID { + garage_net::NodeID::from_slice(bytes.as_slice()).unwrap() } } @@ -140,3 +153,25 @@ pub fn fasthash(data: &[u8]) -> FastHash { pub fn gen_uuid() -> Uuid { rand::thread_rng().gen::<[u8; 32]>().into() } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_increment() { + let zero: FixedBytes32 = [0u8; 32].into(); + let mut one: FixedBytes32 = [0u8; 32].into(); + one.0[31] = 1; + let max: FixedBytes32 = [0xFFu8; 32].into(); + assert_eq!(zero.increment(), Some(one)); + assert_eq!(max.increment(), None); + + let mut test: FixedBytes32 = [0u8; 32].into(); + let i = 0x198DF97209F8FFFFu64; + test.0[24..32].copy_from_slice(&u64::to_be_bytes(i)); + let mut test2: FixedBytes32 = [0u8; 32].into(); + test2.0[24..32].copy_from_slice(&u64::to_be_bytes(i + 1)); + assert_eq!(test.increment(), Some(test2)); + } +} diff --git a/src/util/encode.rs b/src/util/encode.rs index 1cd3198f..c6815d49 100644 --- a/src/util/encode.rs +++ b/src/util/encode.rs @@ -1,28 +1,26 @@ use serde::{Deserialize, Serialize}; -/// Serialize to MessagePacki, without versionning -/// (see garage_util::migrate for functions that manage versionned +/// Serialize to MessagePack, without versioning +/// (see garage_util::migrate for functions that manage versioned /// data formats) pub fn nonversioned_encode(val: &T) -> Result, rmp_serde::encode::Error> where T: Serialize + ?Sized, { let mut wr = Vec::with_capacity(128); - let mut se = rmp_serde::Serializer::new(&mut wr) - .with_struct_map() - .with_string_variants(); + let mut se = rmp_serde::Serializer::new(&mut wr).with_struct_map(); val.serialize(&mut se)?; Ok(wr) } -/// Deserialize from MessagePacki, without versionning -/// (see garage_util::migrate for functions that manage versionned +/// Deserialize from MessagePack, without versioning +/// (see garage_util::migrate for functions that manage versioned /// data formats) pub fn nonversioned_decode(bytes: &[u8]) -> Result where T: for<'de> Deserialize<'de> + ?Sized, { - rmp_serde::decode::from_read_ref::<_, T>(bytes) + rmp_serde::decode::from_slice::<_>(bytes) } /// Serialize to JSON, truncating long result diff --git a/src/util/error.rs b/src/util/error.rs index 3fcee71d..170d2687 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -2,7 +2,7 @@ use std::fmt; use std::io; -use err_derive::Error; +use thiserror::Error; use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; @@ -12,64 +12,61 @@ use crate::encode::debug_serialize; /// Regroup all Garage errors #[derive(Debug, Error)] pub enum Error { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] io::Error), + #[error("IO error: {0}")] + Io(#[from] io::Error), - #[error(display = "Hyper error: {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Hyper error: {0}")] + Hyper(#[from] hyper::Error), - #[error(display = "HTTP error: {}", _0)] - Http(#[error(source)] http::Error), + #[error("HTTP error: {0}")] + Http(#[from] http::Error), - #[error(display = "Invalid HTTP header value: {}", _0)] - HttpHeader(#[error(source)] http::header::ToStrError), + #[error("Invalid HTTP header value: {0}")] + HttpHeader(#[from] http::header::ToStrError), - #[error(display = "Netapp error: {}", _0)] - Netapp(#[error(source)] netapp::error::Error), + #[error("Network error: {0}")] + Net(#[from] garage_net::error::Error), - #[error(display = "DB error: {}", _0)] - Db(#[error(source)] garage_db::Error), + #[error("DB error: {0}")] + Db(#[from] garage_db::Error), - #[error(display = "Messagepack encode error: {}", _0)] - RmpEncode(#[error(source)] rmp_serde::encode::Error), - #[error(display = "Messagepack decode error: {}", _0)] - RmpDecode(#[error(source)] rmp_serde::decode::Error), - #[error(display = "JSON error: {}", _0)] - Json(#[error(source)] serde_json::error::Error), - #[error(display = "TOML decode error: {}", _0)] - TomlDecode(#[error(source)] toml::de::Error), + #[error("Messagepack encode error: {0}")] + RmpEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RmpDecode(#[from] rmp_serde::decode::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::error::Error), + #[error("TOML decode error: {0}")] + TomlDecode(#[from] toml::de::Error), - #[error(display = "Tokio join error: {}", _0)] - TokioJoin(#[error(source)] tokio::task::JoinError), + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), - #[error(display = "Tokio semaphore acquire error: {}", _0)] - TokioSemAcquire(#[error(source)] tokio::sync::AcquireError), + #[error("Tokio semaphore acquire error: {0}")] + TokioSemAcquire(#[from] tokio::sync::AcquireError), - #[error(display = "Tokio broadcast receive error: {}", _0)] - TokioBcastRecv(#[error(source)] tokio::sync::broadcast::error::RecvError), + #[error("Tokio broadcast receive error: {0}")] + TokioBcastRecv(#[from] tokio::sync::broadcast::error::RecvError), - #[error(display = "Remote error: {}", _0)] + #[error("Remote error: {0}")] RemoteError(String), - #[error(display = "Timeout")] + #[error("Timeout")] Timeout, - #[error( - display = "Could not reach quorum of {}. {} of {} request succeeded, others returned errors: {:?}", - _0, - _1, - _2, - _3 - )] - Quorum(usize, usize, usize, Vec), + #[error("Could not reach quorum of {0} (sets={1:?}). {2} of {3} request succeeded, others returned errors: {4:?}")] + Quorum(usize, Option, usize, usize, Vec), - #[error(display = "Unexpected RPC message: {}", _0)] + #[error("Unexpected RPC message: {0}")] UnexpectedRpcMessage(String), - #[error(display = "Corrupt data: does not match hash {:?}", _0)] + #[error("Corrupt data: does not match hash {0:?}")] CorruptData(Hash), - #[error(display = "{}", _0)] + #[error("Missing block {0:?}: no node returned a valid block")] + MissingBlock(Hash), + + #[error("{0}")] Message(String), } diff --git a/src/util/forwarded_headers.rs b/src/util/forwarded_headers.rs index 6ae275aa..12f76434 100644 --- a/src/util/forwarded_headers.rs +++ b/src/util/forwarded_headers.rs @@ -13,7 +13,7 @@ pub fn handle_forwarded_for_headers(headers: &HeaderMap) -> Result< .to_str() .ok_or_message("Error parsing X-Forwarded-For header")?; - let client_ip = IpAddr::from_str(&forwarded_for_ip_str) + let client_ip = IpAddr::from_str(forwarded_for_ip_str) .ok_or_message("Valid IP address not found in X-Forwarded-For header")?; Ok(client_ip.to_string()) diff --git a/src/util/lib.rs b/src/util/lib.rs index c9110fb2..8b035ff0 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -3,18 +3,17 @@ #[macro_use] extern crate tracing; -pub mod async_hash; pub mod background; pub mod config; pub mod crdt; pub mod data; pub mod encode; pub mod error; -pub mod formater; pub mod forwarded_headers; pub mod metrics; pub mod migrate; pub mod persister; +pub mod socket_address; pub mod time; pub mod tranquilizer; pub mod version; diff --git a/src/util/migrate.rs b/src/util/migrate.rs index 1229fd9c..45147c74 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -19,7 +19,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { fn decode(bytes: &[u8]) -> Option { let marker_len = Self::VERSION_MARKER.len(); if bytes.get(..marker_len) == Some(Self::VERSION_MARKER) { - if let Ok(value) = rmp_serde::decode::from_read_ref::<_, Self>(&bytes[marker_len..]) { + if let Ok(value) = rmp_serde::decode::from_slice::<_>(&bytes[marker_len..]) { return Some(value); } } @@ -27,13 +27,11 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { Self::Previous::decode(bytes).map(Self::migrate) } - /// Encode this type with optionnal version marker + /// Encode this type with optional version marker fn encode(&self) -> Result, rmp_serde::encode::Error> { let mut wr = Vec::with_capacity(128); wr.extend_from_slice(Self::VERSION_MARKER); - let mut se = rmp_serde::Serializer::new(&mut wr) - .with_struct_map() - .with_string_variants(); + let mut se = rmp_serde::Serializer::new(&mut wr).with_struct_map(); self.serialize(&mut se)?; Ok(wr) } diff --git a/src/util/socket_address.rs b/src/util/socket_address.rs new file mode 100644 index 00000000..f01225f6 --- /dev/null +++ b/src/util/socket_address.rs @@ -0,0 +1,44 @@ +use std::fmt::{Debug, Display, Formatter}; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::str::FromStr; + +use serde::de::Error; +use serde::{Deserialize, Deserializer}; + +#[derive(Debug, Clone)] +pub enum UnixOrTCPSocketAddress { + TCPSocket(SocketAddr), + UnixSocket(PathBuf), +} + +impl Display for UnixOrTCPSocketAddress { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + UnixOrTCPSocketAddress::TCPSocket(address) => write!(formatter, "http://{}", address), + UnixOrTCPSocketAddress::UnixSocket(path) => { + write!(formatter, "http+unix://{}", path.to_string_lossy()) + } + } + } +} + +impl<'de> Deserialize<'de> for UnixOrTCPSocketAddress { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let string = String::deserialize(deserializer)?; + let string = string.as_str(); + + if string.starts_with("/") { + Ok(UnixOrTCPSocketAddress::UnixSocket( + PathBuf::from_str(string).map_err(Error::custom)?, + )) + } else { + Ok(UnixOrTCPSocketAddress::TCPSocket( + SocketAddr::from_str(string).map_err(Error::custom)?, + )) + } + } +} diff --git a/src/util/version.rs b/src/util/version.rs index 2b2ea271..19907ed1 100644 --- a/src/util/version.rs +++ b/src/util/version.rs @@ -1,18 +1,14 @@ use std::sync::Arc; -use arc_swap::{ArcSwap, ArcSwapOption}; +use arc_swap::ArcSwapOption; lazy_static::lazy_static! { - static ref VERSION: ArcSwap<&'static str> = ArcSwap::new(Arc::new(git_version::git_version!( - prefix = "git:", - cargo_prefix = "cargo:", - fallback = "unknown" - ))); + static ref VERSION: ArcSwapOption<&'static str> = ArcSwapOption::new(None); static ref FEATURES: ArcSwapOption<&'static [&'static str]> = ArcSwapOption::new(None); } pub fn garage_version() -> &'static str { - &VERSION.load() + VERSION.load().as_ref().unwrap() } pub fn garage_features() -> Option<&'static [&'static str]> { @@ -20,7 +16,7 @@ pub fn garage_features() -> Option<&'static [&'static str]> { } pub fn init_version(version: &'static str) { - VERSION.store(Arc::new(version)); + VERSION.store(Some(Arc::new(version))); } pub fn init_features(features: &'static [&'static str]) { diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index d0a23af4..e0cb317f 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.8.2" +version = "1.3.1" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" @@ -14,18 +14,20 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api = { version = "0.8.2", path = "../api" } -garage_model = { version = "0.8.2", path = "../model" } -garage_util = { version = "0.8.2", path = "../util" } -garage_table = { version = "0.8.2", path = "../table" } +garage_api_common.workspace = true +garage_api_s3.workspace = true +garage_model.workspace = true +garage_util.workspace = true +garage_table.workspace = true -err-derive = "0.3" -tracing = "0.1" -percent-encoding = "2.1.0" +thiserror.workspace = true +tracing.workspace = true +percent-encoding.workspace = true -futures = "0.3" +http.workspace = true +http-body-util.workspace = true +hyper.workspace = true -http = "0.2" -hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } +tokio.workspace = true -opentelemetry = "0.17" +opentelemetry.workspace = true diff --git a/src/web/error.rs b/src/web/error.rs index bd8f17b5..aef74923 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -1,31 +1,31 @@ -use err_derive::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; -use garage_api::generic_server::ApiError; +use garage_api_common::generic_server::ApiError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { /// An error received from the API crate - #[error(display = "API error: {}", _0)] - ApiError(garage_api::s3::error::Error), + #[error("API error: {0}")] + ApiError(garage_api_s3::error::Error), /// The file does not exist - #[error(display = "Not found")] + #[error("Not found")] NotFound, /// The client sent a request without host, or with unsupported method - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), } impl From for Error where - garage_api::s3::error::Error: From, + garage_api_s3::error::Error: From, { fn from(err: T) -> Self { - Error::ApiError(garage_api::s3::error::Error::from(err)) + Error::ApiError(garage_api_s3::error::Error::from(err)) } } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 0c7edf23..ea02ab0f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,12 +1,14 @@ -use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc}; +use std::fs::{self, Permissions}; +use std::os::unix::prelude::PermissionsExt; +use std::sync::Arc; -use futures::future::Future; +use tokio::net::{TcpListener, UnixListener}; +use tokio::sync::watch; use hyper::{ - header::{HeaderValue, HOST}, - server::conn::AddrStream, - service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, + body::Incoming as IncomingBody, + header::{HeaderValue, HOST, LOCATION}, + Method, Request, Response, StatusCode, }; use opentelemetry::{ @@ -18,19 +20,26 @@ use opentelemetry::{ use crate::error::*; -use garage_api::helpers::{authority_to_host, host_to_bucket}; -use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket}; -use garage_api::s3::error::{ +use garage_api_common::cors::{ + add_cors_headers, find_matching_cors_rule, handle_options_for_bucket, +}; +use garage_api_common::generic_server::{server_loop, UnixListenerOn}; +use garage_api_common::helpers::*; +use garage_api_s3::error::{ CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError, }; -use garage_api::s3::get::{handle_get, handle_head}; +use garage_api_s3::get::{handle_get_without_ctx, handle_head_without_ctx}; +use garage_api_s3::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; use garage_model::garage::Garage; use garage_table::*; +use garage_util::config::WebConfig; +use garage_util::data::Uuid; use garage_util::error::Error as GarageError; use garage_util::forwarded_headers; use garage_util::metrics::{gen_trace_id, RecordDuration}; +use garage_util::socket_address::UnixOrTCPSocketAddress; struct WebMetrics { request_counter: Counter, @@ -62,61 +71,80 @@ pub struct WebServer { garage: Arc, metrics: Arc, root_domain: String, + add_host_to_metrics: bool, } impl WebServer { /// Run a web server - pub async fn run( - garage: Arc, - addr: SocketAddr, - root_domain: String, - shutdown_signal: impl Future, - ) -> Result<(), GarageError> { + pub fn new(garage: Arc, config: &WebConfig) -> Arc { let metrics = Arc::new(WebMetrics::new()); - let web_server = Arc::new(WebServer { + Arc::new(WebServer { garage, metrics, - root_domain, - }); + root_domain: config.root_domain.clone(), + add_host_to_metrics: config.add_host_to_metrics, + }) + } - let service = make_service_fn(|conn: &AddrStream| { - let web_server = web_server.clone(); + pub async fn run( + self: Arc, + bind_addr: UnixOrTCPSocketAddress, + must_exit: watch::Receiver, + ) -> Result<(), GarageError> { + let server_name = "Web".into(); + info!("Web server listening on {}", bind_addr); - let client_addr = conn.remote_addr(); - async move { - Ok::<_, Error>(service_fn(move |req: Request| { - let web_server = web_server.clone(); + match bind_addr { + UnixOrTCPSocketAddress::TCPSocket(addr) => { + let listener = TcpListener::bind(addr).await?; - web_server.handle_request(req, client_addr) - })) + let handler = + move |stream, socketaddr| self.clone().handle_request(stream, socketaddr); + server_loop(server_name, listener, handler, must_exit).await } - }); + UnixOrTCPSocketAddress::UnixSocket(ref path) => { + if path.exists() { + fs::remove_file(path)? + } - let server = Server::bind(&addr).serve(service); - let graceful = server.with_graceful_shutdown(shutdown_signal); - info!("Web server listening on http://{}", addr); + let listener = UnixListener::bind(path)?; + let listener = UnixListenerOn(listener, path.display().to_string()); - graceful.await?; - Ok(()) + fs::set_permissions(path, Permissions::from_mode(0o222))?; + + let handler = + move |stream, socketaddr| self.clone().handle_request(stream, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + } } async fn handle_request( self: Arc, - req: Request, - addr: SocketAddr, - ) -> Result, Infallible> { + req: Request, + addr: String, + ) -> Result>, http::Error> { + let host_header = req + .headers() + .get(HOST) + .and_then(|x| x.to_str().ok()) + .unwrap_or("") + .to_string(); + if let Ok(forwarded_for_ip_addr) = - forwarded_headers::handle_forwarded_for_headers(&req.headers()) + forwarded_headers::handle_forwarded_for_headers(req.headers()) { + // uri() below has a preceding '/', so no space with host info!( - "{} (via {}) {} {}", + "{} (via {}) {} {}{}", forwarded_for_ip_addr, addr, req.method(), + host_header, req.uri() ); } else { - info!("{} {} {}", addr, req.method(), req.uri()); + info!("{} {} {}{}", addr, req.method(), host_header, req.uri()); } // Lots of instrumentation @@ -125,12 +153,18 @@ impl WebServer { .span_builder(format!("Web {} request", req.method())) .with_trace_id(gen_trace_id()) .with_attributes(vec![ + KeyValue::new("host", format!("{}", host_header.clone())), KeyValue::new("method", format!("{}", req.method())), KeyValue::new("uri", req.uri().to_string()), ]) .start(&tracer); - let metrics_tags = &[KeyValue::new("method", req.method().to_string())]; + let mut metrics_tags = vec![KeyValue::new("method", req.method().to_string())]; + if self.add_host_to_metrics { + metrics_tags.push(KeyValue::new("host", host_header.clone())); + } + + let req = req.map(|_| ()); // The actual handler let res = self @@ -145,30 +179,50 @@ impl WebServer { // Returning the result match res { Ok(res) => { - debug!("{} {} {}", req.method(), res.status(), req.uri()); - Ok(res) + debug!( + "{} {} {}{}", + req.method(), + res.status(), + host_header, + req.uri() + ); + Ok(res + .map(|body| BoxBody::new(http_body_util::BodyExt::map_err(body, Error::from)))) } Err(error) => { info!( - "{} {} {} {}", + "{} {} {}{} {}", req.method(), error.http_status_code(), + host_header, req.uri(), error ); - self.metrics.error_counter.add( - 1, - &[ - metrics_tags[0].clone(), - KeyValue::new("status_code", error.http_status_code().to_string()), - ], - ); + metrics_tags.push(KeyValue::new( + "status_code", + error.http_status_code().to_string(), + )); + self.metrics.error_counter.add(1, &metrics_tags); Ok(error_to_res(error)) } } } - async fn serve_file(self: &Arc, req: &Request) -> Result, Error> { + async fn check_key_exists(self: &Arc, bucket_id: Uuid, key: &str) -> Result { + let exists = self + .garage + .object_table + .get(&bucket_id, &key.to_string()) + .await? + .map(|object| object.versions().iter().any(|v| v.is_data())) + .unwrap_or(false); + Ok(exists) + } + + async fn serve_file( + self: &Arc, + req: &Request<()>, + ) -> Result>, Error> { // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() @@ -191,14 +245,13 @@ impl WebServer { // Check bucket isn't deleted and has website access enabled let bucket = self .garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NotFound)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await + .map_err(|_| Error::NotFound)?; + let bucket_params = bucket.state.into_option().unwrap(); - let website_config = bucket - .params() - .ok_or(Error::NotFound)? + let website_config = bucket_params .website_config .get() .as_ref() @@ -207,22 +260,57 @@ impl WebServer { // Get path let path = req.uri().path().to_string(); let index = &website_config.index_document; - let key = path_to_key(&path, index)?; + let (key, may_redirect) = path_to_keys(&path, index)?; debug!( - "Selected bucket: \"{}\" {:?}, selected key: \"{}\"", - bucket_name, bucket_id, key + "Selected bucket: \"{}\" {:?}, target key: \"{}\", may redirect to: {:?}", + bucket_name, bucket_id, key, may_redirect ); let ret_doc = match *req.method() { - Method::OPTIONS => handle_options_for_bucket(req, &bucket), - Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await, - Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await, + Method::OPTIONS => handle_options_for_bucket(req, &bucket_params) + .map_err(ApiError::from) + .map(|res| res.map(|_empty_body: EmptyBody| empty_body())), + Method::HEAD => { + handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await + } + Method::GET => { + handle_get_without_ctx( + self.garage.clone(), + req, + bucket_id, + &key, + None, + Default::default(), + ) + .await + } _ => Err(ApiError::bad_request("HTTP method not supported")), - } - .map_err(Error::from); + }; - match ret_doc { + // Try implicit redirect on error + let ret_doc_with_redir = match (&ret_doc, may_redirect) { + (Err(ApiError::NoSuchKey), ImplicitRedirect::To { key, url }) + if self.check_key_exists(bucket_id, key.as_str()).await? => + { + Ok(Response::builder() + .status(StatusCode::FOUND) + .header(LOCATION, url) + .body(empty_body()) + .unwrap()) + } + (Ok(ret), _) if ret.headers().contains_key(X_AMZ_WEBSITE_REDIRECT_LOCATION) => { + let redirect_location = ret.headers().get(X_AMZ_WEBSITE_REDIRECT_LOCATION).unwrap(); + Ok(Response::builder() + .status(StatusCode::MOVED_PERMANENTLY) + .header(LOCATION, redirect_location) + .body(empty_body()) + .unwrap()) + } + _ => ret_doc, + }; + + match ret_doc_with_redir.map_err(Error::from) { Err(error) => { // For a HEAD or OPTIONS method, and for non-4xx errors, // we don't return the error document as content, @@ -245,10 +333,18 @@ impl WebServer { // Create a fake HTTP request with path = the error document let req2 = Request::builder() .uri(format!("http://{}/{}", host, &error_document)) - .body(Body::empty()) + .body(()) .unwrap(); - match handle_get(self.garage.clone(), &req2, bucket_id, &error_document, None).await + match handle_get_without_ctx( + self.garage.clone(), + &req2, + bucket_id, + &error_document, + None, + Default::default(), + ) + .await { Ok(mut error_doc) => { // The error won't be logged back in handle_request, @@ -283,7 +379,7 @@ impl WebServer { } Ok(mut resp) => { // Maybe add CORS headers - if let Some(rule) = find_matching_cors_rule(&bucket, req)? { + if let Some(rule) = find_matching_cors_rule(&bucket_params, req)? { add_cors_headers(&mut resp, rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } @@ -293,7 +389,7 @@ impl WebServer { } } -fn error_to_res(e: Error) -> Response { +fn error_to_res(e: Error) -> Response> { // If we are here, it is either that: // - there was an error before trying to get the requested URL // from the bucket (e.g. bucket not found) @@ -301,37 +397,72 @@ fn error_to_res(e: Error) -> Response { // was a HEAD request or we couldn't get the error document) // We do NOT enter this code path when returning the bucket's // error document (this is handled in serve_file) - let body = Body::from(format!("{}\n", e)); - let mut http_error = Response::new(body); + let mut body_str = format!( + r"{http_code} {code_text} +

{http_code} {code_text}

", + http_code = e.http_status_code().as_u16(), + code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"), + ); + if let Error::ApiError(ref err) = e { + body_str.push_str(&format!( + r" +
    +
  • Code: {s3_code}
  • +
  • Message: {s3_message}.
  • +
", + s3_code = err.aws_code(), + s3_message = err, + )); + } + let mut http_error = Response::new(string_body(body_str)); *http_error.status_mut() = e.http_status_code(); e.add_headers(http_error.headers_mut()); + http_error.headers_mut().insert( + http::header::CONTENT_TYPE, + "text/html; charset=utf-8".parse().unwrap(), + ); http_error } +#[derive(Debug, PartialEq)] +enum ImplicitRedirect { + No, + To { key: String, url: String }, +} + /// Path to key /// /// Convert the provided path to the internal key /// When a path ends with "/", we append the index name to match traditional web server behavior /// which is also AWS S3 behavior. -fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { +/// +/// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html +fn path_to_keys<'a>(path: &'a str, index: &str) -> Result<(String, ImplicitRedirect), Error> { let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?; - if !path_utf8.starts_with('/') { - return Err(Error::BadRequest("Path must start with a / (slash)".into())); - } + let base_key = match path_utf8.strip_prefix("/") { + Some(bk) => bk, + None => return Err(Error::BadRequest("Path must start with a / (slash)".into())), + }; + let is_bucket_root = base_key.len() == 0; + let is_trailing_slash = path_utf8.ends_with("/"); - match path_utf8.chars().last() { - None => unreachable!(), - Some('/') => { - let mut key = String::with_capacity(path_utf8.len() + index.len()); - key.push_str(&path_utf8[1..]); - key.push_str(index); - Ok(key.into()) - } - Some(_) => match path_utf8 { - Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), - Cow::Owned(pu8) => Ok(pu8[1..].to_string().into()), - }, + match (is_bucket_root, is_trailing_slash) { + // It is not possible to store something at the root of the bucket (ie. empty key), + // the only option is to fetch the index + (true, _) => Ok((index.to_string(), ImplicitRedirect::No)), + + // "If you create a folder structure in your bucket, you must have an index document at each level. In each folder, the index document must have the same name, for example, index.html. When a user specifies a URL that resembles a folder lookup, the presence or absence of a trailing slash determines the behavior of the website. For example, the following URL, with a trailing slash, returns the photos/index.html index document." + (false, true) => Ok((format!("{base_key}{index}"), ImplicitRedirect::No)), + + // "However, if you exclude the trailing slash from the preceding URL, Amazon S3 first looks for an object photos in the bucket. If the photos object is not found, it searches for an index document, photos/index.html. If that document is found, Amazon S3 returns a 302 Found message and points to the photos/ key. For subsequent requests to photos/, Amazon S3 returns photos/index.html. If the index document is not found, Amazon S3 returns an error." + (false, false) => Ok(( + base_key.to_string(), + ImplicitRedirect::To { + key: format!("{base_key}/{index}"), + url: format!("{path}/"), + }, + )), } } @@ -340,13 +471,37 @@ mod tests { use super::*; #[test] - fn path_to_key_test() -> Result<(), Error> { - assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg"); - assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html"); - assert_eq!(path_to_key("/", "index.html")?, "index.html"); - assert_eq!(path_to_key("/hello", "index.html")?, "hello"); - assert!(path_to_key("", "index.html").is_err()); - assert!(path_to_key("i/am/relative", "index.html").is_err()); + fn path_to_keys_test() -> Result<(), Error> { + assert_eq!( + path_to_keys("/file%20.jpg", "index.html")?, + ( + "file .jpg".to_string(), + ImplicitRedirect::To { + key: "file .jpg/index.html".to_string(), + url: "/file%20.jpg/".to_string() + } + ) + ); + assert_eq!( + path_to_keys("/%20t/", "index.html")?, + (" t/index.html".to_string(), ImplicitRedirect::No) + ); + assert_eq!( + path_to_keys("/", "index.html")?, + ("index.html".to_string(), ImplicitRedirect::No) + ); + assert_eq!( + path_to_keys("/hello", "index.html")?, + ( + "hello".to_string(), + ImplicitRedirect::To { + key: "hello/index.html".to_string(), + url: "/hello/".to_string() + } + ) + ); + assert!(path_to_keys("", "index.html").is_err()); + assert!(path_to_keys("i/am/relative", "index.html").is_err()); Ok(()) } }