Improve clarity

.drone.yml

@@ -0,0 +1,632 @@
+---
+kind: pipeline
+name: default
+
+workspace:
+  base: /drone/garage
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  HOME: /drone/garage
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link shell.nix --arg release false  -A inputDerivation
+
+  - name: code quality
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-shell --arg release false --run "cargo fmt -- --check"
+      - nix-shell --arg release false --run "cargo clippy -- --allow clippy::needless_borrow --allow  clippy::needless-return --deny warnings"
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
+
+  - name: unit tests
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - |
+        nix-build \
+          --no-build-output \
+          --argstr target x86_64-unknown-linux-musl \
+          --argstr compileMode test
+      - ./result*/bin/garage_api*
+      - ./result*/bin/garage_model*
+      - ./result*/bin/garage_rpc*
+      - ./result*/bin/garage_table*
+      - ./result*/bin/garage_util*
+      - ./result*/bin/garage_web*
+      - ./result*/bin/garage*
+
+  - name: smoke-test
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
+      - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-build --no-out-link shell.nix --arg release false -A inputDerivation))
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --argstr compileMode test))
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --arg release false))
+    when:
+      event:
+      - cron
+
+trigger:
+  event:
+  - custom
+  - push
+  - pull_request
+  - tag
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+name: website
+
+steps:
+  - name: build
+    image: hrektts/mdbook
+    commands:
+      - cd doc/book
+      - mdbook build
+
+  - name: upload
+    image: plugins/s3
+    settings:
+      bucket: garagehq.deuxfleurs.fr
+      access_key:
+        from_secret: garagehq_aws_access_key_id
+      secret_key:
+        from_secret: garagehq_aws_secret_access_key
+      source: doc/book/book/**/*
+      strip_prefix: doc/book/book/
+      target: /
+      path_style: true
+      endpoint: https://garage.deuxfleurs.fr
+      region: garage
+    when:
+      event:
+        - push
+      branch:
+        - main
+      repo:
+        - Deuxfleurs/garage
+
+trigger:
+  event:
+  - custom
+  - push
+  - pull_request
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-x86_64
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: x86_64-unknown-linux-musl
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+
+  - name: integration
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/amd64"
+      CONTAINER_NAME: "dxflrs/amd64_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-i686
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: i686-unknown-linux-musl
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+
+  - name: integration
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/386"
+      CONTAINER_NAME: "dxflrs/386_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-aarch64
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: aarch64-unknown-linux-musl
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link ./shell.nix --arg rust false --arg integration false -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/arm64"
+      CONTAINER_NAME: "dxflrs/arm64_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-armv6l
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: armv6l-unknown-linux-musleabihf
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg integration false --arg rust false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/arm"
+      CONTAINER_NAME: "dxflrs/arm_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: refresh-release-page
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+
+steps:
+  - name: refresh-index
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
+      - nix-shell --arg integration false --arg rust false --run "refresh_index"
+
+depends_on:
+  - release-linux-x86_64
+  - release-linux-i686
+  - release-linux-aarch64
+  - release-linux-armv6l
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: signature
+hmac: 1c33490cc2902564c4250a409c156683d0d549b8c9d5aee4e46d1bde4e0ccf2c
+
+...

.envrc

@@ -1 +0,0 @@
-use flake

.gitignore

@@ -3,4 +3,3 @@
 /pki
 **/*.rs.bk
 *.swp
-/.direnv

.woodpecker/debug.yaml

@@ -1,44 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - push
-  - tag
-  - pull_request
-  - deployment
-  - cron
-  - manual
-
-steps:
-  - name: check formatting
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.fmt
-
-  - name: build
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.dev
-
-  - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-lmdb
-
-  - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-sqlite
-
-  - name: unit + func tests (fjall)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-fjall
-
-  - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.dev
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-    depends_on: [ build ]

.woodpecker/publish.yaml

@@ -1,33 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - deployment
-  - cron
-
-depends_on:
-  - release
-
-steps:
-  - name: refresh-index
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-    commands:
-      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --attr ci --run "refresh_index"
-
-  - name: multiarch-docker
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "multiarch_docker"

.woodpecker/release.yaml

@@ -1,71 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - deployment
-  - cron
-
-matrix:
-  include:
-    - ARCH: amd64
-      TARGET: x86_64-unknown-linux-musl
-    - ARCH: i386
-      TARGET: i686-unknown-linux-musl
-    - ARCH: arm64
-      TARGET: aarch64-unknown-linux-musl
-    - ARCH: arm
-      TARGET: armv6l-unknown-linux-musleabihf
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-
-  - name: check is static binary
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
-
-  - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-    when:
-      - matrix:
-          ARCH: amd64
-      - matrix:
-          ARCH: i386
-
-  - name: upgrade tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
-    when:
-      - matrix:
-          ARCH: amd64
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      TARGET: "${TARGET}"
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-    commands:
-      - nix-shell --attr ci --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      DOCKER_PLATFORM: "linux/${ARCH}"
-      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
-      DOCKER_AUTH:
-        from_secret: docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "to_docker"

Cargo.toml

@@ -1,154 +1,16 @@
 [workspace]
-resolver = "2"
 members = [
-	"src/db",
 	"src/util",
-	"src/net",
 	"src/rpc",
 	"src/table",
-	"src/block",
 	"src/model",
-	"src/api/common",
+	"src/api",
-	"src/api/s3",
-	"src/api/k2v",
-	"src/api/admin",
 	"src/web",
 	"src/garage",
-	"src/k2v-client",
-	"src/format-table",
 ]
 
-default-members = ["src/garage"]
+[profile.dev]
-
+lto = "off"
-[workspace.dependencies]
-
-# Internal Garage crates
-format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.3.1", path = "src/api/common" }
-garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
-garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
-garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
-garage_block = { version = "1.3.1", path = "src/block" }
-garage_db = { version = "1.3.1", path = "src/db", default-features = false }
-garage_model = { version = "1.3.1", path = "src/model", default-features = false }
-garage_net = { version = "1.3.1", path = "src/net" }
-garage_rpc = { version = "1.3.1", path = "src/rpc" }
-garage_table = { version = "1.3.1", path = "src/table" }
-garage_util = { version = "1.3.1", path = "src/util" }
-garage_web = { version = "1.3.1", path = "src/web" }
-k2v-client = { version = "0.0.4", path = "src/k2v-client" }
-
-# External crates from crates.io
-arc-swap = "1.0"
-argon2 = "0.5"
-async-trait = "0.1.7"
-backtrace = "0.3"
-base64 = "0.21"
-blake2 = "0.10"
-bytes = "1.0"
-bytesize = "1.1"
-cfg-if = "1.0"
-chrono = "0.4"
-crc32fast = "1.4"
-crc32c = "0.6"
-crypto-common = "0.1"
-gethostname = "0.4"
-git-version = "0.3.4"
-hex = "0.4"
-hexdump = "0.1"
-html-escape = "0.2.13"
-hmac = "0.12"
-itertools = "0.12"
-ipnet = "2.9.0"
-lazy_static = "1.4"
-md-5 = "0.10"
-mktemp = "0.5"
-nix = { version = "0.29", default-features = false, features = ["fs"] }
-nom = "7.1"
-parking_lot = "0.12"
-parse_duration = "2.1"
-pin-project = "1.0.12"
-pnet_datalink = "0.34"
-rand = "0.8"
-sha1 = "0.10"
-sha2 = "0.10"
-timeago = { version = "0.4", default-features = false }
-xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
-
-aes-gcm = { version = "0.10", features = ["aes", "stream"] }
-sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
-kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
-
-clap = { version = "4.1", features = ["derive", "env"] }
-pretty_env_logger = "0.5"
-structopt = { version = "0.3", default-features = false }
-syslog-tracing = "0.3"
-tracing = "0.1"
-tracing-journald = "0.3.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-
-heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.37"
-r2d2 = "0.8"
-r2d2_sqlite = "0.31"
-fjall = "2.4"
-
-async-compression = { version = "0.4", features = ["tokio", "zstd"] }
-zstd = { version = "0.13", default-features = false }
-
-quick-xml = { version = "0.26", features = [ "serialize" ] }
-rmp-serde = "1.1.2"
-serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
-serde_bytes = "0.11"
-serde_json = "1.0"
-toml = { version = "0.8", default-features = false, features = ["parse"] }
-
-# newer version requires rust edition 2021
-k8s-openapi = { version = "0.21", features = ["v1_24"] }
-kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] }
-schemars = "0.8"
-reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] }
-
-form_urlencoded = "1.0.0"
-http = "1.0"
-httpdate = "1.0"
-http-range = "0.1"
-http-body-util = "0.1"
-hyper = { version = "1.0", default-features = false }
-hyper-util = { version = "0.1", features = [ "full" ] }
-multer = "3.0"
-percent-encoding = "2.2"
-roxmltree = "0.19"
-url = "2.3"
-
-futures = "0.3"
-futures-util = "0.3"
-tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
-tokio-util = { version = "0.7", features = ["compat", "io"] }
-tokio-stream = { version = "0.1", features = ["net"] }
-
-opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
-opentelemetry-prometheus = "0.10"
-opentelemetry-otlp = "0.10"
-opentelemetry-contrib = "0.9"
-prometheus = "0.13"
-
-# used by the k2v-client crate only
-aws-sigv4 = { version = "1.1", default-features = false }
-hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
-log = "0.4"
-thiserror = "2.0"
-
-# ---- used only as build / dev dependencies ----
-assert-json-diff = "2.0"
-rustc_version = "0.4.0"
-static_init = "1.0"
-aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
-aws-sdk-config = { version = "1.62", default-features = false }
-aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
 [profile.release]
-lto = "thin"
+debug = true
-codegen-units = 16
-opt-level = 3
-strip = "debuginfo"

Dockerfile

@@ -4,4 +4,4 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-CMD [ "/garage", "server"]
+CMD [ "/garage", "server", "-c", "config.toml"]

Makefile

@@ -1,22 +1,13 @@
-.PHONY: doc all run1 run2 run3
+.PHONY: doc all release shell
 
 all:
-	clear
+	clear; cargo build
-	cargo build
 
-# ----
+doc:
+	cd doc/book; mdbook build
 
-run1:
+release:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config1.toml server
+	nix-build --arg release true
-run1rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config1.toml server
 
-run2:
+shell:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config2.toml server
+	nix-shell
-run2rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config2.toml server
-
-run3:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config3.toml server
-run3rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config3.toml server

README.md

@@ -1,38 +1,26 @@
-Garage [![status-badge](https://woodpecker.deuxfleurs.fr/api/badges/1/status.svg)](https://woodpecker.deuxfleurs.fr/repos/1)
+Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
 ===
 
 <p align="center" style="text-align:center;">
 	<a href="https://garagehq.deuxfleurs.fr">
-	<img alt="Garage logo" src="https://garagehq.deuxfleurs.fr/img/logo.svg" height="200" />
+	<img alt="Garage logo" src="doc/logo/garage.png" height="200" />
 	</a>
 </p>
 
-<p align="center" style="text-align:center;">
+Garage is a lightweight S3-compatible distributed object store, with the following goals:
-	[ <strong><a href="https://garagehq.deuxfleurs.fr/">Website and documentation</a></strong>
-	| <a href="https://garagehq.deuxfleurs.fr/_releases.html">Binary releases</a>
-	| <a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">Git repository</a>
-	| <a href="https://matrix.to/#/%23garage:deuxfleurs.fr">Matrix channel</a>
-	]
-</p>
 
-Garage is an S3-compatible distributed object storage service
+- As self-contained as possible
-designed for self-hosting at a small-to-medium scale.
+- Easy to set up
+- Highly resilient to network failures, network latency, disk failures, sysadmin failures
+- Relatively simple
+- Made for multi-datacenter deployments
 
-Garage is designed for storage clusters composed of nodes running
+Non-goals include:
-at different physical locations,
-in order to easily provide a storage service that replicates data at these different
-locations and stays available even when some servers are unreachable.
-Garage also focuses on being lightweight, easy to operate, and highly resilient to
-machine failures.
 
-Garage is built by [Deuxfleurs](https://deuxfleurs.fr),
+- Extremely high performance
-an experimental small-scale self hosted service provider,
+- Complete implementation of the S3 API
-which has been using it in production since its first release in 2020.
+- Erasure coding (our replication model is simply to copy the data as is on several nodes, in different datacenters if possible)
 
-Learn more on our dedicated documentation pages:
+Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
 
-- [Goals and use cases](https://garagehq.deuxfleurs.fr/documentation/design/goals/)
+**[Go to the documentation](https://garagehq.deuxfleurs.fr)**
-- [Features](https://garagehq.deuxfleurs.fr/documentation/reference-manual/features/)
-- [Quick start](https://garagehq.deuxfleurs.fr/documentation/quick-start/)
-
-Garage is entirely free software released under the terms of the AGPLv3.

TODO

@@ -0,0 +1,27 @@
+Testing
+-------
+
+How are we going to test that our replication method works correctly?
+We will have to introduce lots of dummy data and then add/remove nodes many times.
+
+
+Attaining S3 compatibility
+--------------------------
+
+- test multipart uploads
+- get ranges
+
+- fix sync not working in some cases ? (when starting from empty?)
+
+- api_server following the S3 semantics for head/get/put/list/delete: verify more that it works as intended
+- PUT requests: verify content-md5 if provided
+- possibly other necessary endpoints ?
+
+
+Lower priority
+--------------
+
+- less a priority: hinted handoff
+- repair: re-propagate block ref table to rc
+- FIXME in rpc_server when garage shuts down and futures can be interrupted
+  (tokio::spawn should be replaced by a new function background::spawn_joinable)

default.nix

@@ -1,24 +1,75 @@
-{ system ? builtins.currentSystem, git_version ? null, }:
+{
+  system ? builtins.currentSystem,
+  release ? false,
+  target ? "x86_64-unknown-linux-musl",
+  compileMode ? null,
+  git_version ? null,
+}:
 
 with import ./nix/common.nix;
 
 let
-  pkgs = import nixpkgs { };
+  crossSystem = { config = target; };
-  compile = import ./nix/compile.nix;
+in let
 
-  build_release = target: (compile {
+  pkgs = import pkgsSrc {
-    inherit target system git_version nixpkgs;
+    inherit system crossSystem;
-    crane = flake.inputs.crane;
+    overlays = [ cargo2nixOverlay rustOverlay ];
-    rust-overlay = flake.inputs.rust-overlay;
-    release = true;
-  }).garage;
-
-in {
-  releasePackages = {
-    amd64 = build_release "x86_64-unknown-linux-musl";
-    i386 = build_release "i686-unknown-linux-musl";
-    arm64 = build_release "aarch64-unknown-linux-musl";
-    arm = build_release "armv6l-unknown-linux-musleabihf";
   };
-  flakePackages = flake.packages.${system};
+  rustDist = pkgs.buildPackages.rust-bin.stable.latest.default;
-}
+
+  /*
+   The following complexity should be abstracted by makePackageSet' (note the final quote).
+   However its code uses deprecated features of rust-overlay that can lead to bug.
+   Instead, we build our own rustChannel object with the recommended API of rust-overlay.
+   */
+  rustChannel = rustDist // {
+    cargo = rustDist;
+    rustc = rustDist.override {
+      targets = [
+        (pkgs.rustBuilder.rustLib.realHostTriple pkgs.stdenv.targetPlatform)
+      ];
+    };
+  };
+
+  overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage";
+      overrideAttrs = drv: if git_version != null then {
+        preConfigure = ''
+          ${drv.preConfigure or ""}
+          export GIT_VERSION="${git_version}"
+        '';
+      } else {};
+    })
+  ];
+
+  packageFun = import ./Cargo.nix;
+
+  rustPkgs = pkgs.rustBuilder.makePackageSet {
+    inherit packageFun rustChannel release;
+    packageOverrides = overrides;
+
+    buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
+      inherit rustChannel packageFun;
+      packageOverrides = overrides;
+    };
+
+    localPatterns = [
+      /*
+       The way the default rules are written make think we match recursively, on full path, but the rules are misleading.
+       In fact, the regex is only called on root elements of the crate (and not recursively).
+       This behavior does not work well with our nested modules.
+       We tried to build a "deny list" but negative lookup ahead are not supported on Nix.
+       As a workaround, we have to register all our submodules in this allow list...
+       */
+       ''^(src|tests)''          # fixed default
+       ''.*\.(rs|toml)$''        # fixed default
+       ''^(crdt|replication|cli)''   # our crate submodules
+    ];
+  };
+
+in
+  if compileMode == "test"
+    then builtins.mapAttrs (name: value: rustPkgs.workspace.${name} { inherit compileMode; }) rustPkgs.workspace
+    else rustPkgs.workspace.garage { inherit compileMode; }

doc/api/README.md

@@ -1,17 +0,0 @@
-# Browse doc
-
-Run in this directory:
-
-```
-python3 -m http.server 
-```
-
-And open in your browser:
-  - http://localhost:8000/garage-admin-v0.html
-
-# Validate doc
-
-```
-wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.1.0/openapi-generator-cli-6.1.0.jar -O openapi-generator-cli.jar
-java -jar openapi-generator-cli.jar validate -i garage-admin-v0.yml
-```

doc/api/css/redoc.css

@@ -1,59 +0,0 @@
-/* montserrat-300 - latin */
-@font-face {
-  font-family: 'Montserrat';
-  font-style: normal;
-  font-weight: 300;
-  src: local(''),
-       url('../fonts/montserrat-v25-latin-300.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/montserrat-v25-latin-300.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-
-/* montserrat-regular - latin */
-@font-face {
-  font-family: 'Montserrat';
-  font-style: normal;
-  font-weight: 400;
-  src: local(''),
-       url('../fonts/montserrat-v25-latin-regular.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/montserrat-v25-latin-regular.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-
-/* montserrat-700 - latin */
-@font-face {
-  font-family: 'Montserrat';
-  font-style: normal;
-  font-weight: 700;
-  src: local(''),
-       url('../fonts/montserrat-v25-latin-700.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/montserrat-v25-latin-700.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-/* roboto-300 - latin */
-@font-face {
-  font-family: 'Roboto';
-  font-style: normal;
-  font-weight: 300;
-  src: local(''),
-       url('../fonts/roboto-v30-latin-300.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/roboto-v30-latin-300.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-
-/* roboto-regular - latin */
-@font-face {
-  font-family: 'Roboto';
-  font-style: normal;
-  font-weight: 400;
-  src: local(''),
-       url('../fonts/roboto-v30-latin-regular.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/roboto-v30-latin-regular.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-
-/* roboto-700 - latin */
-@font-face {
-  font-family: 'Roboto';
-  font-style: normal;
-  font-weight: 700;
-  src: local(''),
-       url('../fonts/roboto-v30-latin-700.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
-       url('../fonts/roboto-v30-latin-700.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
-}
-