fix blocksync not resyncing compressed blocs properly

set_extension don't behave verywell with extensions containing multiple dots

.drone.yml

@@ -0,0 +1,134 @@
+---
+kind: pipeline
+name: default
+
+workspace:
+  base: /drone/garage
+
+volumes:
+- name: cargo_home
+  temp: {}
+
+steps:
+  - name: restore-cache
+    image: meltwater/drone-cache:dev
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+    pull: true
+    settings:
+      restore: true
+      archive_format: "gzip"
+      bucket: drone-cache
+      cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
+      region: garage
+      mount:
+        - '/drone/cargo'
+        - 'target'
+      path_style: true
+      endpoint: https://garage.deuxfleurs.fr
+    when:
+      branch:
+      - nonexistent_skip_this_step
+
+  - name: build
+    image: lxpz/garage_builder_amd64:1
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      CARGO_HOME: /drone/cargo
+    commands:
+      - pwd
+      - cargo fmt -- --check
+      - cargo build
+
+  - name: cargo-test
+    image: lxpz/garage_builder_amd64:1
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      CARGO_HOME: /drone/cargo
+    commands:
+      - cargo test
+
+  - name: rebuild-cache
+    image: meltwater/drone-cache:dev
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+    pull: true
+    settings:
+      rebuild: true
+      archive_format: "gzip"
+      bucket: drone-cache
+      cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
+      region: garage
+      mount:
+        - '/drone/cargo'
+        - 'target'
+      path_style: true
+      endpoint: https://garage.deuxfleurs.fr
+    when:
+      branch:
+      - nonexistent_skip_this_step
+
+  - name: smoke-test
+    image: lxpz/garage_builder_amd64:1
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      CARGO_HOME: /drone/cargo
+    commands:
+      - ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+---
+kind: pipeline
+name: website
+
+steps:
+  - name: build
+    image: hrektts/mdbook
+    commands:
+      - cd doc/book
+      - mdbook build
+
+  - name: upload
+    image: plugins/s3
+    settings:
+      bucket: garagehq.deuxfleurs.fr
+      access_key:
+        from_secret: garagehq_aws_access_key_id
+      secret_key:
+        from_secret: garagehq_aws_secret_access_key
+      source: doc/book/book/**/*
+      strip_prefix: doc/book/book/
+      target: /
+      path_style: true
+      endpoint: https://garage.deuxfleurs.fr
+      region: garage
+    when:
+      event:
+        - push
+      branch:
+        - main
+      repo:
+        - Deuxfleurs/garage
+
+---
+kind: signature
+hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d
+
+...

.envrc

@@ -1 +0,0 @@
-use flake

.gitignore

@@ -3,4 +3,3 @@
 /pki
 **/*.rs.bk
 *.swp
-/.direnv

.woodpecker/debug.yaml

@@ -1,44 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - push
-  - tag
-  - pull_request
-  - deployment
-  - cron
-  - manual
-
-steps:
-  - name: check formatting
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.fmt
-
-  - name: build
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.dev
-
-  - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-lmdb
-
-  - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-sqlite
-
-  - name: unit + func tests (fjall)
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.tests-fjall
-
-  - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build -j4 --attr flakePackages.dev
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-    depends_on: [ build ]

.woodpecker/publish.yaml

@@ -1,33 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - deployment
-  - cron
-
-depends_on:
-  - release
-
-steps:
-  - name: refresh-index
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-    commands:
-      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --attr ci --run "refresh_index"
-
-  - name: multiarch-docker
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "multiarch_docker"

.woodpecker/release.yaml

@@ -1,71 +0,0 @@
-labels:
-  nix: "enabled"
-
-when:
-  event:
-  - deployment
-  - cron
-
-matrix:
-  include:
-    - ARCH: amd64
-      TARGET: x86_64-unknown-linux-musl
-    - ARCH: i386
-      TARGET: i686-unknown-linux-musl
-    - ARCH: arm64
-      TARGET: aarch64-unknown-linux-musl
-    - ARCH: arm
-      TARGET: armv6l-unknown-linux-musleabihf
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-
-  - name: check is static binary
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
-
-  - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-    when:
-      - matrix:
-          ARCH: amd64
-      - matrix:
-          ARCH: i386
-
-  - name: upgrade tests
-    image: nixpkgs/nix:nixos-24.05
-    commands:
-      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
-    when:
-      - matrix:
-          ARCH: amd64
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      TARGET: "${TARGET}"
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-    commands:
-      - nix-shell --attr ci --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-24.05
-    environment:
-      DOCKER_PLATFORM: "linux/${ARCH}"
-      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
-      DOCKER_AUTH:
-        from_secret: docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "to_docker"

Cargo.toml

@@ -1,154 +1,16 @@
 [workspace]
-resolver = "2"
 members = [
-	"src/db",
 	"src/util",
-	"src/net",
 	"src/rpc",
 	"src/table",
-	"src/block",
 	"src/model",
-	"src/api/common",
-	"src/api/s3",
-	"src/api/k2v",
-	"src/api/admin",
+	"src/api",
 	"src/web",
 	"src/garage",
-	"src/k2v-client",
-	"src/format-table",
 ]
 
-default-members = ["src/garage"]
-
-[workspace.dependencies]
-
-# Internal Garage crates
-format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.3.1", path = "src/api/common" }
-garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
-garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
-garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
-garage_block = { version = "1.3.1", path = "src/block" }
-garage_db = { version = "1.3.1", path = "src/db", default-features = false }
-garage_model = { version = "1.3.1", path = "src/model", default-features = false }
-garage_net = { version = "1.3.1", path = "src/net" }
-garage_rpc = { version = "1.3.1", path = "src/rpc" }
-garage_table = { version = "1.3.1", path = "src/table" }
-garage_util = { version = "1.3.1", path = "src/util" }
-garage_web = { version = "1.3.1", path = "src/web" }
-k2v-client = { version = "0.0.4", path = "src/k2v-client" }
-
-# External crates from crates.io
-arc-swap = "1.0"
-argon2 = "0.5"
-async-trait = "0.1.7"
-backtrace = "0.3"
-base64 = "0.21"
-blake2 = "0.10"
-bytes = "1.0"
-bytesize = "1.1"
-cfg-if = "1.0"
-chrono = "0.4"
-crc32fast = "1.4"
-crc32c = "0.6"
-crypto-common = "0.1"
-gethostname = "0.4"
-git-version = "0.3.4"
-hex = "0.4"
-hexdump = "0.1"
-html-escape = "0.2.13"
-hmac = "0.12"
-itertools = "0.12"
-ipnet = "2.9.0"
-lazy_static = "1.4"
-md-5 = "0.10"
-mktemp = "0.5"
-nix = { version = "0.29", default-features = false, features = ["fs"] }
-nom = "7.1"
-parking_lot = "0.12"
-parse_duration = "2.1"
-pin-project = "1.0.12"
-pnet_datalink = "0.34"
-rand = "0.8"
-sha1 = "0.10"
-sha2 = "0.10"
-timeago = { version = "0.4", default-features = false }
-xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
-
-aes-gcm = { version = "0.10", features = ["aes", "stream"] }
-sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
-kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
-
-clap = { version = "4.1", features = ["derive", "env"] }
-pretty_env_logger = "0.5"
-structopt = { version = "0.3", default-features = false }
-syslog-tracing = "0.3"
-tracing = "0.1"
-tracing-journald = "0.3.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-
-heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.37"
-r2d2 = "0.8"
-r2d2_sqlite = "0.31"
-fjall = "2.4"
-
-async-compression = { version = "0.4", features = ["tokio", "zstd"] }
-zstd = { version = "0.13", default-features = false }
-
-quick-xml = { version = "0.26", features = [ "serialize" ] }
-rmp-serde = "1.1.2"
-serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
-serde_bytes = "0.11"
-serde_json = "1.0"
-toml = { version = "0.8", default-features = false, features = ["parse"] }
-
-# newer version requires rust edition 2021
-k8s-openapi = { version = "0.21", features = ["v1_24"] }
-kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] }
-schemars = "0.8"
-reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] }
-
-form_urlencoded = "1.0.0"
-http = "1.0"
-httpdate = "1.0"
-http-range = "0.1"
-http-body-util = "0.1"
-hyper = { version = "1.0", default-features = false }
-hyper-util = { version = "0.1", features = [ "full" ] }
-multer = "3.0"
-percent-encoding = "2.2"
-roxmltree = "0.19"
-url = "2.3"
-
-futures = "0.3"
-futures-util = "0.3"
-tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
-tokio-util = { version = "0.7", features = ["compat", "io"] }
-tokio-stream = { version = "0.1", features = ["net"] }
-
-opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
-opentelemetry-prometheus = "0.10"
-opentelemetry-otlp = "0.10"
-opentelemetry-contrib = "0.9"
-prometheus = "0.13"
-
-# used by the k2v-client crate only
-aws-sigv4 = { version = "1.1", default-features = false }
-hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
-log = "0.4"
-thiserror = "2.0"
-
-# ---- used only as build / dev dependencies ----
-assert-json-diff = "2.0"
-rustc_version = "0.4.0"
-static_init = "1.0"
-aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
-aws-sdk-config = { version = "1.62", default-features = false }
-aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
+[profile.dev]
+lto = "off"
 
 [profile.release]
-lto = "thin"
-codegen-units = 16
-opt-level = 3
-strip = "debuginfo"
+debug = true

Dockerfile

@@ -1,7 +1,10 @@
-FROM scratch
+FROM archlinux:latest
 
+RUN mkdir -p /garage/meta
+RUN mkdir -p /garage/data
 ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
-COPY result/bin/garage /
-CMD [ "/garage", "server"]
+COPY target/release/garage.stripped /garage/garage
+
+CMD /garage/garage server -c /garage/config.toml

Makefile

@@ -1,22 +1,20 @@
-.PHONY: doc all run1 run2 run3
+BIN=target/release/garage
+DOCKER=lxpz/garage_amd64
 
 all:
-	clear
-	cargo build
+	clear; cargo build
 
-# ----
+$(BIN):
+	RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
 
-run1:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config1.toml server
-run1rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config1.toml server
+$(BIN).stripped: $(BIN)
+	cp $^ $@
+	strip $@
 
-run2:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config2.toml server
-run2rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config2.toml server
-
-run3:
-	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config3.toml server
-run3rel:
-	RUST_LOG=garage=debug ./target/release/garage -c tmp/config3.toml server
+docker: $(BIN).stripped
+	docker pull archlinux:latest
+	docker build -t $(DOCKER):$(TAG) .
+	docker push $(DOCKER):$(TAG)
+	docker tag $(DOCKER):$(TAG) $(DOCKER):latest
+	docker push $(DOCKER):latest
+	

README.md

@@ -1,38 +1,26 @@
-Garage [![status-badge](https://woodpecker.deuxfleurs.fr/api/badges/1/status.svg)](https://woodpecker.deuxfleurs.fr/repos/1)
+Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
 ===
 
 <p align="center" style="text-align:center;">
 	<a href="https://garagehq.deuxfleurs.fr">
-	<img alt="Garage logo" src="https://garagehq.deuxfleurs.fr/img/logo.svg" height="200" />
+	<img alt="Garage logo" src="doc/logo/garage.png" height="200" />
 	</a>
 </p>
 
-<p align="center" style="text-align:center;">
-	[ <strong><a href="https://garagehq.deuxfleurs.fr/">Website and documentation</a></strong>
-	| <a href="https://garagehq.deuxfleurs.fr/_releases.html">Binary releases</a>
-	| <a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">Git repository</a>
-	| <a href="https://matrix.to/#/%23garage:deuxfleurs.fr">Matrix channel</a>
-	]
-</p>
+Garage is a lightweight S3-compatible distributed object store, with the following goals:
 
-Garage is an S3-compatible distributed object storage service
-designed for self-hosting at a small-to-medium scale.
+- As self-contained as possible
+- Easy to set up
+- Highly resilient to network failures, network latency, disk failures, sysadmin failures
+- Relatively simple
+- Made for multi-datacenter deployments
 
-Garage is designed for storage clusters composed of nodes running
-at different physical locations,
-in order to easily provide a storage service that replicates data at these different
-locations and stays available even when some servers are unreachable.
-Garage also focuses on being lightweight, easy to operate, and highly resilient to
-machine failures.
+Non-goals include:
 
-Garage is built by [Deuxfleurs](https://deuxfleurs.fr),
-an experimental small-scale self hosted service provider,
-which has been using it in production since its first release in 2020.
+- Extremely high performance
+- Complete implementation of the S3 API
+- Erasure coding (our replication model is simply to copy the data as is on several nodes, in different datacenters if possible)
 
-Learn more on our dedicated documentation pages:
+Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
 
-- [Goals and use cases](https://garagehq.deuxfleurs.fr/documentation/design/goals/)
-- [Features](https://garagehq.deuxfleurs.fr/documentation/reference-manual/features/)
-- [Quick start](https://garagehq.deuxfleurs.fr/documentation/quick-start/)
-
-Garage is entirely free software released under the terms of the AGPLv3.
+**[Go to the documentation](https://garagehq.deuxfleurs.fr)**

TODO

@@ -0,0 +1,27 @@
+Testing
+-------
+
+How are we going to test that our replication method works correctly?
+We will have to introduce lots of dummy data and then add/remove nodes many times.
+
+
+Attaining S3 compatibility
+--------------------------
+
+- test multipart uploads
+- get ranges
+
+- fix sync not working in some cases ? (when starting from empty?)
+
+- api_server following the S3 semantics for head/get/put/list/delete: verify more that it works as intended
+- PUT requests: verify content-md5 if provided
+- possibly other necessary endpoints ?
+
+
+Lower priority
+--------------
+
+- less a priority: hinted handoff
+- repair: re-propagate block ref table to rc
+- FIXME in rpc_server when garage shuts down and futures can be interrupted
+  (tokio::spawn should be replaced by a new function background::spawn_joinable)

config.dev.toml

@@ -0,0 +1,22 @@
+block_size = 1048576			# objects are split in blocks of maximum this number of bytes
+
+metadata_dir = "/tmp/garage-meta"
+data_dir = "/tmp/garage-data"
+
+rpc_bind_addr = "[::]:3901"		# the port other Garage nodes will use to talk to this node
+
+bootstrap_peers = []
+
+max_concurrent_rpc_requests = 12
+data_replication_factor = 3
+meta_replication_factor = 3
+meta_epidemic_fanout = 3
+
+[s3_api]
+api_bind_addr = "[::1]:3900"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
+s3_region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
+
+[s3_web]
+bind_addr = "[::1]:3902"
+root_domain = ".garage.tld"
+index = "index.html"

default.nix

@@ -1,24 +0,0 @@
-{ system ? builtins.currentSystem, git_version ? null, }:
-
-with import ./nix/common.nix;
-
-let
-  pkgs = import nixpkgs { };
-  compile = import ./nix/compile.nix;
-
-  build_release = target: (compile {
-    inherit target system git_version nixpkgs;
-    crane = flake.inputs.crane;
-    rust-overlay = flake.inputs.rust-overlay;
-    release = true;
-  }).garage;
-
-in {
-  releasePackages = {
-    amd64 = build_release "x86_64-unknown-linux-musl";
-    i386 = build_release "i686-unknown-linux-musl";
-    arm64 = build_release "aarch64-unknown-linux-musl";
-    arm = build_release "armv6l-unknown-linux-musleabihf";
-  };
-  flakePackages = flake.packages.${system};
-}

doc/20201202_talk/.gitignore

@@ -0,0 +1,12 @@
+*
+
+!img
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!talk.pdf

doc/20201202_talk/Makefile

@@ -0,0 +1,6 @@
+talk.pdf: talk.tex img/garage_distributed.pdf img/consistent_hashing_1.pdf img/consistent_hashing_2.pdf img/consistent_hashing_3.pdf img/consistent_hashing_4.pdf img/garage_tables.pdf
+	pdflatex talk.tex
+	
+img/%.pdf: img/%.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+