mirror of
https://git.deuxfleurs.fr/Deuxfleurs/garage.git
synced 2026-11-10 16:19:15 -05:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
0e05b9bc77 |
||
|
|
163ee977a0 |
||
|
|
307858bf0a | ||
|
|
76efb4999d |
||
|
|
6e28ea6bdc |
||
|
|
7ba80b6108 |
||
|
|
e2e8ea4df5 | ||
|
992284d545 | ||
|
|
d4fd074000 | ||
|
|
aeadb071c6 | ||
|
|
4d5263ccf3 | ||
|
|
e4b66a9e28 | ||
|
|
9215d63e3a | ||
|
|
01c623e391 | ||
|
|
de8e47435f | ||
|
|
7560a41ab1 |
||
|
|
898fd1dc81 |
||
|
|
af6774a511 |
||
|
|
c35c472dc9 | ||
|
|
718ae00548 | ||
|
|
c3bd672d58 |
||
|
|
93a9f96130 |
||
|
|
7380f3855c | ||
|
|
6cbc8d6ec9 | ||
|
|
c8906f200b | ||
|
|
653d3d588f | ||
|
|
a74eccfd6e | ||
|
|
ee9b685994 | ||
|
|
92d54770bb | ||
|
|
30bec0758b | ||
|
|
b476b702c8 | ||
|
|
7d3a951836 | ||
|
|
f4bf987627 | ||
|
|
b3b0b20d72 | ||
|
|
47d0aee9f8 | ||
|
|
6e0cb2dfb6 | ||
|
|
c4c4b7dedc |
||
|
|
8225fa2e4b | ||
|
|
ab67bd88de |
||
|
|
0f192a96b5 |
||
|
|
7b85056942 |
||
|
|
7fd1f9a869 |
||
|
|
c5d8dc7d6d |
||
|
|
fa11cb746a |
||
|
|
f11bd80d2a |
||
|
|
595dc0ed0d |
||
|
|
78eeaab5ed |
||
|
|
22fbb3b892 |
||
|
|
0eb5baea1a |
||
|
|
7d772737a5 | ||
|
|
4c2d8f5a96 | ||
|
|
0325086dac | ||
|
|
700925263f |
||
|
|
6c83f66700 | ||
|
|
55e4a93bad | ||
|
9cb9945131 | ||
|
|
7f9c1d5595 | ||
|
|
991cf1b818 | ||
|
|
e26cb2640d | ||
|
|
22041e924b | ||
|
|
b119e9d3c4 | ||
|
3e6534d7a8 | ||
|
|
2dae4a25d6 | ||
|
|
fcd566e89d |
61 changed files with 1370 additions and 505 deletions
59
.drone.yml
59
.drone.yml
|
|
@ -1,12 +1,20 @@
|
|||
---
|
||||
kind: pipeline
|
||||
name: default
|
||||
|
||||
workspace:
|
||||
base: /drone/garage
|
||||
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
temp: {}
|
||||
|
||||
steps:
|
||||
- name: restore-cache
|
||||
image: meltwater/drone-cache:dev
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: cache_aws_access_key_id
|
||||
|
|
@ -20,35 +28,41 @@ steps:
|
|||
cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
|
||||
region: garage
|
||||
mount:
|
||||
- '/drone/cargo'
|
||||
- 'target'
|
||||
- '/drone/cargo/registry/index'
|
||||
- '/drone/cargo/registry/cache'
|
||||
- '/drone/cargo/bin'
|
||||
- '/drone/cargo/git/db'
|
||||
path_style: true
|
||||
endpoint: https://garage.deuxfleurs.fr
|
||||
when:
|
||||
branch:
|
||||
- nonexistent_skip_this_step
|
||||
|
||||
- name: build
|
||||
image: rust:buster
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
environment:
|
||||
CARGO_HOME: /drone/cargo
|
||||
commands:
|
||||
- apt-get update
|
||||
- apt-get install --yes libsodium-dev
|
||||
- pwd
|
||||
- cargo fmt -- --check
|
||||
- cargo build
|
||||
|
||||
- name: cargo-test
|
||||
image: rust:buster
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
environment:
|
||||
CARGO_HOME: /drone/cargo
|
||||
commands:
|
||||
- apt-get update
|
||||
- apt-get install --yes libsodium-dev
|
||||
- cargo test
|
||||
|
||||
- name: rebuild-cache
|
||||
image: meltwater/drone-cache:dev
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: cache_aws_access_key_id
|
||||
|
|
@ -62,22 +76,22 @@ steps:
|
|||
cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
|
||||
region: garage
|
||||
mount:
|
||||
- '/drone/cargo'
|
||||
- 'target'
|
||||
- '/drone/cargo/registry/index'
|
||||
- '/drone/cargo/registry/cache'
|
||||
- '/drone/cargo/git/db'
|
||||
- '/drone/cargo/bin'
|
||||
path_style: true
|
||||
endpoint: https://garage.deuxfleurs.fr
|
||||
when:
|
||||
branch:
|
||||
- nonexistent_skip_this_step
|
||||
|
||||
- name: smoke-test
|
||||
image: rust:buster
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
environment:
|
||||
CARGO_HOME: /drone/cargo
|
||||
commands:
|
||||
- apt-get update
|
||||
- apt-get install --yes libsodium-dev awscli python-pip
|
||||
- pip install s3cmd
|
||||
- ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
---
|
||||
|
|
@ -106,6 +120,15 @@ steps:
|
|||
endpoint: https://garage.deuxfleurs.fr
|
||||
region: garage
|
||||
when:
|
||||
event:
|
||||
- push
|
||||
branch:
|
||||
- main
|
||||
repo:
|
||||
- Deuxfleurs/garage
|
||||
|
||||
---
|
||||
kind: signature
|
||||
hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d
|
||||
|
||||
...
|
||||
|
|
|
|||
130
Cargo.lock
generated
130
Cargo.lock
generated
|
|
@ -95,6 +95,9 @@ name = "cc"
|
|||
version = "1.0.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
|
|
@ -221,6 +224,12 @@ dependencies = [
|
|||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastcdc"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
|
|
@ -365,7 +374,7 @@ dependencies = [
|
|||
"hex",
|
||||
"log",
|
||||
"pretty_env_logger",
|
||||
"rand",
|
||||
"rand 0.8.3",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"sled",
|
||||
|
|
@ -383,6 +392,7 @@ dependencies = [
|
|||
"chrono",
|
||||
"crypto-mac 0.10.0",
|
||||
"err-derive",
|
||||
"fastcdc",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_model",
|
||||
|
|
@ -397,6 +407,7 @@ dependencies = [
|
|||
"log",
|
||||
"md-5",
|
||||
"percent-encoding",
|
||||
"rand 0.7.3",
|
||||
"roxmltree",
|
||||
"sha2",
|
||||
"tokio",
|
||||
|
|
@ -415,12 +426,13 @@ dependencies = [
|
|||
"garage_util",
|
||||
"hex",
|
||||
"log",
|
||||
"rand",
|
||||
"rand 0.8.3",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"sled",
|
||||
"tokio",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -459,7 +471,7 @@ dependencies = [
|
|||
"garage_util",
|
||||
"hexdump",
|
||||
"log",
|
||||
"rand",
|
||||
"rand 0.8.3",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
|
|
@ -479,7 +491,7 @@ dependencies = [
|
|||
"http",
|
||||
"hyper",
|
||||
"log",
|
||||
"rand",
|
||||
"rand 0.8.3",
|
||||
"rmp-serde",
|
||||
"rustls",
|
||||
"serde",
|
||||
|
|
@ -529,6 +541,17 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.9.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.2"
|
||||
|
|
@ -537,7 +560,7 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -760,6 +783,15 @@ version = "0.4.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.49"
|
||||
|
|
@ -1043,6 +1075,19 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||
dependencies = [
|
||||
"getrandom 0.1.16",
|
||||
"libc",
|
||||
"rand_chacha 0.2.2",
|
||||
"rand_core 0.5.1",
|
||||
"rand_hc 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.3"
|
||||
|
|
@ -1050,9 +1095,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
"rand_chacha 0.3.0",
|
||||
"rand_core 0.6.2",
|
||||
"rand_hc 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1062,7 +1117,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||
dependencies = [
|
||||
"getrandom 0.1.16",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1071,7 +1135,16 @@ version = "0.6.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
|
||||
dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1080,7 +1153,7 @@ version = "0.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1581,6 +1654,12 @@ dependencies = [
|
|||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
|
|
@ -1703,3 +1782,32 @@ name = "xxhash-rust"
|
|||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.6.1+zstd.1.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "3.0.1+zstd.1.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "1.4.20+zstd.1.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -2,7 +2,7 @@ BIN=target/release/garage
|
|||
DOCKER=lxpz/garage_amd64
|
||||
|
||||
all:
|
||||
clear; RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --no-default-features
|
||||
clear; cargo build
|
||||
|
||||
$(BIN):
|
||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Garage [](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
|
||||
Garage [](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
|
||||
===
|
||||
|
||||
<p align="center" style="text-align:center;">
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
- [Host a website](./cookbook/website.md)
|
||||
- [Integrate as a media backend]()
|
||||
- [Operate a cluster]()
|
||||
- [Recovering from failures](./cookbook/recovering.md)
|
||||
|
||||
- [Reference Manual](./reference_manual/index.md)
|
||||
- [Garage CLI]()
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
|
||||
A cookbook, when you cook, is a collection of recipes.
|
||||
Similarly, Garage's cookbook contains a collection of recipes that are known to works well!
|
||||
This chapter could also be referred as "Tutorials" or "Best practises".
|
||||
This chapter could also be referred as "Tutorials" or "Best practices".
|
||||
|
|
|
|||
99
doc/book/src/cookbook/recovering.md
Normal file
99
doc/book/src/cookbook/recovering.md
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
# Recovering from failures
|
||||
|
||||
Garage is meant to work on old, second-hand hardware.
|
||||
In particular, this makes it likely that some of your drives will fail, and some manual intervention will be needed.
|
||||
Fear not! For Garage is fully equipped to handle drive failures, in most common cases.
|
||||
|
||||
## A note on availability of Garage
|
||||
|
||||
With nodes dispersed in 3 datacenters or more, here are the guarantees Garage provides with the default replication strategy (3 copies of all data, which is the recommended value):
|
||||
|
||||
- The cluster remains fully functional as long as the machines that fail are in only one datacenter. This includes a whole datacenter going down due to power/Internet outage.
|
||||
- No data is lost as long as the machines that fail are in at most two datacenters.
|
||||
|
||||
Of course this only works if your Garage nodes are correctly configured to be aware of the datacenter in which they are located.
|
||||
Make sure this is the case using `garage status` to check on the state of your cluster's configuration.
|
||||
|
||||
|
||||
## First option: removing a node
|
||||
|
||||
If you don't have spare parts (HDD, SDD) to replace the failed component, and if there are enough remaining nodes in your cluster
|
||||
(at least 3), you can simply remove the failed node from Garage's configuration.
|
||||
Note that if you **do** intend to replace the failed parts by new ones, using this method followed by adding back the node is **not recommended** (although it should work),
|
||||
and you should instead use one of the methods detailed in the next sections.
|
||||
|
||||
Removing a node is done with the following command:
|
||||
|
||||
```
|
||||
garage node remove --yes <node_id>
|
||||
```
|
||||
|
||||
(you can get the `node_id` of the failed node by running `garage status`)
|
||||
|
||||
This will repartition the data and ensure that 3 copies of everything are present on the nodes that remain available.
|
||||
|
||||
|
||||
|
||||
## Replacement scenario 1: only data is lost, metadata is fine
|
||||
|
||||
The recommended deployment for Garage uses an SSD to store metadata, and an HDD to store blocks of data.
|
||||
In the case where only a single HDD crashes, the blocks of data are lost but the metadata is still fine.
|
||||
|
||||
This is very easy to recover by setting up a new HDD to replace the failed one.
|
||||
The node does not need to be fully replaced and the configuration doesn't need to change.
|
||||
We just need to tell Garage to get back all the data blocks and store them on the new HDD.
|
||||
|
||||
First, set up a new HDD to store Garage's data directory on the failed node, and restart Garage using
|
||||
the existing configuration. Then, run:
|
||||
|
||||
```
|
||||
garage repair -a --yes blocks
|
||||
```
|
||||
|
||||
This will re-synchronize blocks of data that are missing to the new HDD, reading them from copies located on other nodes.
|
||||
|
||||
You can check on the advancement of this process by doing the following command:
|
||||
|
||||
```
|
||||
garage stats -a
|
||||
```
|
||||
|
||||
Look out for the following output:
|
||||
|
||||
```
|
||||
Block manager stats:
|
||||
resync queue length: 26541
|
||||
```
|
||||
|
||||
This indicates that one of the Garage node is in the process of retrieving missing data from other nodes.
|
||||
This number decreases to zero when the node is fully synchronized.
|
||||
|
||||
|
||||
## Replacement scenario 2: metadata (and possibly data) is lost
|
||||
|
||||
This scenario covers the case where a full node fails, i.e. both the metadata directory and
|
||||
the data directory are lost, as well as the case where only the metadata directory is lost.
|
||||
|
||||
To replace the lost node, we will start from an empty metadata directory, which means
|
||||
Garage will generate a new node ID for the replacement node.
|
||||
We will thus need to remove the previous node ID from Garage's configuration and replace it by the ID of the new node.
|
||||
|
||||
If your data directory is stored on a separate drive and is still fine, you can keep it, but it is not necessary to do so.
|
||||
In all cases, the data will be rebalanced and the replacement node will not store the same pieces of data
|
||||
as were originally stored on the one that failed. So if you keep the data files, the rebalancing
|
||||
might be faster but most of the pieces will be deleted anyway from the disk and replaced by other ones.
|
||||
|
||||
First, set up a new drive to store the metadata directory for the replacement node (a SSD is recommended),
|
||||
and for the data directory if necessary. You can then start Garage on the new node.
|
||||
The restarted node should generate a new node ID, and it should be shown as `NOT CONFIGURED` in `garage status`.
|
||||
The ID of the lost node should be shown in `garage status` in the section for disconnected/unavailable nodes.
|
||||
|
||||
Then, replace the broken node by the new one, using:
|
||||
|
||||
```
|
||||
garage node configure --replace <old_node_id> \
|
||||
-c <capacity> -d <datacenter> -t <node_tag> <new_node_id>
|
||||
```
|
||||
|
||||
Garage will then start synchronizing all required data on the new node.
|
||||
This process can be monitored using the `garage stats -a` command.
|
||||
|
|
@ -7,14 +7,14 @@ We did not test other architecture/operating system but, as long as your archite
|
|||
## From Docker
|
||||
|
||||
Our docker image is currently named `lxpz/garage_amd64` and is stored on the [Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
||||
We encourage you to use a fixed tag (eg. `v0.1.1d`) and not the `latest` tag.
|
||||
For this example, we will use the latest published version at the time of the writing which is `v0.1.1d` but it's up to you
|
||||
We encourage you to use a fixed tag (eg. `v0.2.1`) and not the `latest` tag.
|
||||
For this example, we will use the latest published version at the time of the writing which is `v0.2.1` but it's up to you
|
||||
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
sudo docker pull lxpz/garage_amd64:v0.1.1d
|
||||
sudo docker pull lxpz/garage_amd64:v0.2.1
|
||||
```
|
||||
|
||||
## From source
|
||||
|
|
|
|||
|
|
@ -40,22 +40,18 @@ garagectl key new --name nextcloud-app-key
|
|||
|
||||
You will have the following output (this one is fake, `key_id` and `secret_key` were generated with the openssl CLI tool):
|
||||
|
||||
```javascript
|
||||
Key {
|
||||
key_id: "GK3515373e4c851ebaad366558",
|
||||
secret_key: "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34",
|
||||
name: "nextcloud-app-key",
|
||||
name_timestamp: 1603280506694,
|
||||
deleted: false,
|
||||
authorized_buckets: []
|
||||
}
|
||||
```
|
||||
Key name: nextcloud-app-key
|
||||
Key ID: GK3515373e4c851ebaad366558
|
||||
Secret key: 7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34
|
||||
Authorized buckets:
|
||||
```
|
||||
|
||||
Check that everything works as intended (be careful, info works only with your key identifier and not with its friendly name!):
|
||||
Check that everything works as intended:
|
||||
|
||||
```
|
||||
garagectl key list
|
||||
garagectl key info GK3515373e4c851ebaad366558
|
||||
garagectl key info nextcloud-app-key
|
||||
```
|
||||
|
||||
## Allow a key to access a bucket
|
||||
|
|
@ -67,7 +63,7 @@ garagectl bucket allow \
|
|||
--read \
|
||||
--write
|
||||
nextcloud-bucket \
|
||||
--key GK3515373e4c851ebaad366558
|
||||
--key nextcloud-app-key
|
||||
```
|
||||
|
||||
You can check at any times allowed keys on your bucket with:
|
||||
|
|
|
|||
|
|
@ -11,20 +11,20 @@ As this part is not relevant for a test cluster, you can use this one-liner to c
|
|||
|
||||
```bash
|
||||
garagectl status | grep UNCONFIGURED | grep -Po '^[0-9a-f]+' | while read id; do
|
||||
garagectl node configure -d dc1 -n 10 $id
|
||||
garagectl node configure -d dc1 -c 1 $id
|
||||
done
|
||||
```
|
||||
|
||||
## Real-world cluster
|
||||
|
||||
For our example, we will suppose we have the following infrastructure (Tokens, Identifier and Datacenter are specific values to garage described in the following):
|
||||
For our example, we will suppose we have the following infrastructure (Capacity, Identifier and Datacenter are specific values to garage described in the following):
|
||||
|
||||
| Location | Name | Disk Space | `Tokens` | `Identifier` | `Datacenter` |
|
||||
|----------|---------|------------|----------|--------------|--------------|
|
||||
| Paris | Mercury | 1 To | `100` | `8781c5` | `par1` |
|
||||
| Paris | Venus | 2 To | `200` | `2a638e` | `par1` |
|
||||
| London | Earth | 2 To | `200` | `68143d` | `lon1` |
|
||||
| Brussels | Mars | 1.5 To | `150` | `212f75` | `bru1` |
|
||||
| Location | Name | Disk Space | `Capacity` | `Identifier` | `Datacenter` |
|
||||
|----------|---------|------------|------------|--------------|--------------|
|
||||
| Paris | Mercury | 1 To | `2` | `8781c5` | `par1` |
|
||||
| Paris | Venus | 2 To | `4` | `2a638e` | `par1` |
|
||||
| London | Earth | 2 To | `4` | `68143d` | `lon1` |
|
||||
| Brussels | Mars | 1.5 To | `3` | `212f75` | `bru1` |
|
||||
|
||||
### Identifier
|
||||
|
||||
|
|
@ -45,14 +45,15 @@ garagectl status
|
|||
|
||||
It will display the IP address associated with each node; from the IP address you will be able to recognize the node.
|
||||
|
||||
### Tokens
|
||||
### Capacity
|
||||
|
||||
Garage reasons on an arbitrary metric about disk storage that is named "tokens".
|
||||
The number of tokens must be proportional to the disk space dedicated to the node.
|
||||
Additionaly, ideally the number of tokens must be in the order of magnitude of 100
|
||||
to provide a good trade-off between data load balancing and performances (*this sentence must be verified, it may be wrong*).
|
||||
Garage reasons on an arbitrary metric about disk storage that is named the *capacity* of a node.
|
||||
The capacity configured in Garage must be proportional to the disk space dedicated to the node.
|
||||
Additionaly, the capacity values used in Garage should be as small as possible, with
|
||||
1 ideally representing the size of your smallest server.
|
||||
|
||||
Here we chose 1 token = 10 Go but you are free to select the value that best fit your needs.
|
||||
Here we chose that 1 unit of capacity = 0.5 To, so that we can express servers of size
|
||||
1 To and 2 To, as wel as the intermediate size 1.5 To.
|
||||
|
||||
### Datacenter
|
||||
|
||||
|
|
@ -65,8 +66,8 @@ Behind the scene, garage will try to store the same data on different sites to p
|
|||
Given the information above, we will configure our cluster as follow:
|
||||
|
||||
```
|
||||
garagectl node configure --datacenter par1 -n 100 -t mercury 8781c5
|
||||
garagectl node configure --datacenter par1 -n 200 -t venus 2a638e
|
||||
garagectl node configure --datacenter lon1 -n 200 -t earth 68143d
|
||||
garagectl node configure --datacenter bru1 -n 150 -t mars 212f75
|
||||
garagectl node configure --datacenter par1 -c 2 -t mercury 8781c5
|
||||
garagectl node configure --datacenter par1 -c 4 -t venus 2a638e
|
||||
garagectl node configure --datacenter lon1 -c 4 -t earth 68143d
|
||||
garagectl node configure --datacenter bru1 -c 3 -t mars 212f75
|
||||
```
|
||||
|
|
|
|||
|
|
@ -124,8 +124,8 @@ For our example, we will suppose the following infrastructure:
|
|||
|----------|---------|------------|------------|
|
||||
| Paris | Mercury | fc00:1::1 | 1 To |
|
||||
| Paris | Venus | fc00:1::2 | 2 To |
|
||||
| London | Earth | fc00:1::2 | 2 To |
|
||||
| Brussels | Mars | fc00:B::1 | 1.5 To |
|
||||
| London | Earth | fc00:B::1 | 2 To |
|
||||
| Brussels | Mars | fc00:F::1 | 1.5 To |
|
||||
|
||||
On each machine, we will have a similar setup, especially you must consider the following folders/files:
|
||||
- `/etc/garage/pki`: Garage certificates, must be generated on your computer and copied on the servers
|
||||
|
|
|
|||
|
|
@ -4,63 +4,68 @@
|
|||
</a>
|
||||
</p>
|
||||
|
||||
```
|
||||
This very website is hosted using Garage. In other words: the doc is the PoC!
|
||||
```
|
||||
|
||||
# The Garage Geo-Distributed Data Store
|
||||
|
||||
Garage is a lightweight geo-distributed data store.
|
||||
It comes from the observation that despite numerous object stores
|
||||
many people have broken data management policies (backup/replication on a single site or none at all).
|
||||
To promote better data management policies, with focused on the following desirable properties:
|
||||
To promote better data management policies, we focused on the following **desirable properties**:
|
||||
|
||||
- **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target hyperconverged infrastructures
|
||||
- **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures
|
||||
- **Simple**: simple to understand, simple to operate, simple to debug
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenter, offices, etc.) interconnected through a regular internet connection.
|
||||
- **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
|
||||
- **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures.
|
||||
- **Simple**: simple to understand, simple to operate, simple to debug.
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
|
||||
|
||||
We also noted that the pursuit of some other goals are detrimental to our initial goals.
|
||||
The following have been identified has non-goals, if it matters to you, you should not use Garage:
|
||||
The following has been identified as **non-goals** (if these points matter to you, you should not use Garage):
|
||||
|
||||
- **Extreme performances**: high performances constrain a lot the design and the infrastructure; we seek performances through minimalism only.
|
||||
- **Feature extensiveness**: complete implementation of the S3 API or any other API to make garage a drop-in replacement is not targeted as it could lead to decisions impacting our desirable properties.
|
||||
- **Storage optimizations**: erasure coding or any other coding technique both increase the difficulty of placing data and synchronizing; we limit ourselves to duplication.
|
||||
- **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such syncronizations are translated in network messages that impose severe constraints on the deployment.
|
||||
- **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such synchronizations are translated in network messages that impose severe constraints on the deployment.
|
||||
|
||||
## Supported and planned protocols
|
||||
|
||||
Garage speaks (or will speak) the following protocols:
|
||||
|
||||
- [S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) - *SUPPORTED* - Enable applications to store large blobs such as pictures, video, images, documents, etc. S3 is versatile enough to also be used to publish a static website.
|
||||
- [IMAP](https://github.com/go-pluto/pluto) - *PLANNED* - email storage is quite complex to get good oerformances.
|
||||
To keep performances optimals, most imap servers only support on-disk storage.
|
||||
- [IMAP](https://github.com/go-pluto/pluto) - *PLANNED* - email storage is quite complex to get good performances.
|
||||
To keep performances optimal, most IMAP servers only support on-disk storage.
|
||||
We plan to add logic to Garage to make it a viable solution for email storage.
|
||||
- *More to come*
|
||||
|
||||
## Use Cases
|
||||
|
||||
**[Deuxfleurs](https://deuxfleurs.fr) :** Garage is used by Deuxfleurs which is a non-profit hosting organization.
|
||||
Especially, it is used to host their main website, this documentation and some of its members's blogs. Additionally,
|
||||
Garage is used as a [backend for Nextcloud](https://docs.nextcloud.com/server/20/admin_manual/configuration_files/primary_storage.html). Deuxfleurs also plans to use Garage as their [Matrix's media backend](https://github.com/matrix-org/synapse-s3-storage-provider) and has the backend of [OCIS](https://github.com/owncloud/ocis).
|
||||
**[Deuxfleurs](https://deuxfleurs.fr):** Garage is used by Deuxfleurs which is a non-profit hosting organization.
|
||||
Especially, it is used to host their main website, this documentation and some of its members' blogs.
|
||||
Additionally, Garage is used as a [backend for Nextcloud](https://docs.nextcloud.com/server/20/admin_manual/configuration_files/primary_storage.html).
|
||||
Deuxfleurs also plans to use Garage as their [Matrix's media backend](https://github.com/matrix-org/synapse-s3-storage-provider) and as the backend of [OCIS](https://github.com/owncloud/ocis).
|
||||
|
||||
*Are you using Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add your organization here!*
|
||||
|
||||
## Comparison to existing software
|
||||
|
||||
**[Minio](https://min.io/) :** Minio shares our *self-contained & lightweight* goal but selected two of our non-goals: *storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
|
||||
However, by pursuing these two non-goals, minio do not reach our desirable properties.
|
||||
First, it fails on the *simple* property: due to the erasure coding, minio has severe limitations on how drives can be added or deleted from a cluster.
|
||||
Second, it fails on the *interned enabled* property: due to its strong consistency, minio is latency sensitive.
|
||||
Furthermore, minio has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
|
||||
**[MinIO](https://min.io/):** MinIO shares our *Self-contained & lightweight* goal but selected two of our non-goals: *Storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
|
||||
However, by pursuing these two non-goals, MinIO do not reach our desirable properties.
|
||||
Firstly, it fails on the *Simple* property: due to the erasure coding, MinIO has severe limitations on how drives can be added or deleted from a cluster.
|
||||
Secondly, it fails on the *Internet enabled* property: due to its strong consistency, MinIO is latency sensitive.
|
||||
Furthermore, MinIO has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
|
||||
|
||||
**[Openstack Swift](https://docs.openstack.org/swift/latest/) :**
|
||||
OpenStack Swift at least fails on the *self-contained & lightweight* goal.
|
||||
Starting it requires around 8Gb of RAM, which is too much especially in an hyperconverged infrastructure.
|
||||
It seems also to be far from *Simple*.
|
||||
**[Openstack Swift](https://docs.openstack.org/swift/latest/):**
|
||||
OpenStack Swift at least fails on the *Self-contained & lightweight* goal.
|
||||
Starting it requires around 8GB of RAM, which is too much especially in an hyperconverged infrastructure.
|
||||
We also do not classify Swift as *Simple*.
|
||||
|
||||
**[Ceph](https://ceph.io/ceph-storage/object-storage/) :**
|
||||
**[Ceph](https://ceph.io/ceph-storage/object-storage/):**
|
||||
This review holds for the whole Ceph stack, including the RADOS paper, Ceph Object Storage module, the RADOS Gateway, etc.
|
||||
At is core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
|
||||
makes Ceph latency sensitive and fails our *Internet enabled* goal.
|
||||
Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
|
||||
In a certain way, Ceph and Minio are closer togethers than they are from Garage or OpenStack Swift.
|
||||
At its core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
|
||||
makes Ceph latency-sensitive and fails our *Internet enabled* goal.
|
||||
Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *Self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
|
||||
In a certain way, Ceph and MinIO are closer together than they are from Garage or OpenStack Swift.
|
||||
|
||||
*More comparisons are available in our [Related Work](design/related_work.md) chapter.*
|
||||
|
||||
|
|
@ -71,29 +76,29 @@ We reference here other places on the Internet where you can learn more about Ga
|
|||
|
||||
### Rust API (docs.rs)
|
||||
|
||||
If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code documentation!
|
||||
If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code's documentation!
|
||||
|
||||
- [garage\_api](https://docs.rs/garage_api/latest/garage_api/) - contains the S3 standard API endpoint
|
||||
- [garage\_model](https://docs.rs/garage_model/latest/garage_model/) - contains Garage's model built on the table abstraction
|
||||
- [garage\_rpc](https://docs.rs/garage_rpc/latest/garage_rpc/) - contains Garage's federation protocol
|
||||
- [garage\_table](https://docs.rs/garage_table/latest/garage_table/) - contains core Garage's CRDT datatypes
|
||||
- [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage entrypoints (daemon, cli)
|
||||
- [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage helpers
|
||||
- [garage\_web](https://docs.rs/garage_web/latest/garage_web/) - contains the S3 website endpoint
|
||||
|
||||
### Talks
|
||||
|
||||
We love to talk and hear about Garage, that's why we keep a log here:
|
||||
|
||||
- [(fr, 2020-12-02) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/master/doc/20201202_talk/talk.pdf)
|
||||
- [(fr, 2020-12-02) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/20201202_talk/talk.pdf)
|
||||
|
||||
*Did you write or talk about Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add a link here!*
|
||||
|
||||
## Community
|
||||
|
||||
If you want to discuss with us, you can join our Matrix channel at [#garage:deuxfleurs.fr](https://matrix.to/#/#garage:deuxfleurs.fr).
|
||||
Our code and our issue tracker, which is the place where you should report bugs, are managed on [Deuxfleurs' Gitea](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
Our code repository and issue tracker, which is the place where you should report bugs, is managed on [Deuxfleurs' Gitea](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
|
||||
## License
|
||||
|
||||
Garage, all the source code, is released under the [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html).
|
||||
Garage's source code, is released under the [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html).
|
||||
Please note that if you patch Garage and then use it to provide any service over a network, you must share your code!
|
||||
|
|
|
|||
6
script/builder_image/Dockerfile
Normal file
6
script/builder_image/Dockerfile
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
FROM rust:buster
|
||||
RUN apt-get update && \
|
||||
apt-get install --yes libsodium-dev awscli python-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN rustup component add rustfmt
|
||||
RUN pip install s3cmd
|
||||
8
script/builder_image/Makefile
Normal file
8
script/builder_image/Makefile
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
DOCKER=lxpz/garage_builder_amd64
|
||||
|
||||
docker:
|
||||
docker build -t $(DOCKER):$(TAG) .
|
||||
docker push $(DOCKER):$(TAG)
|
||||
docker tag $(DOCKER):$(TAG) $(DOCKER):latest
|
||||
docker push $(DOCKER):latest
|
||||
|
||||
|
|
@ -35,6 +35,9 @@ data_replication_factor = 3
|
|||
meta_replication_factor = 3
|
||||
meta_epidemic_fanout = 3
|
||||
|
||||
enable_compression = true
|
||||
compressin_level = 10
|
||||
|
||||
[s3_api]
|
||||
api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
||||
s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
|
||||
|
|
|
|||
|
|
@ -23,39 +23,43 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # < INLINE_THRESHOLD = 307
|
|||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5
|
||||
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10
|
||||
|
||||
dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64
|
||||
dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64
|
||||
dd if=/dev/urandom bs=1M count=10 | base64 -w0 > /tmp/garage.3.b64
|
||||
|
||||
echo "s3 api testing..."
|
||||
for idx in $(seq 1 3); do
|
||||
for idx in {1,2,3}.{rnd,b64}; do
|
||||
# AWS sends
|
||||
awsgrg cp /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.aws
|
||||
awsgrg cp /tmp/garage.$idx s3://eprouvette/garage.$idx.aws
|
||||
|
||||
awsgrg ls s3://eprouvette
|
||||
|
||||
awsgrg cp s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
s3grg get s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
awsgrg rm s3://eprouvette/garage.$idx.aws
|
||||
|
||||
# S3CMD sends
|
||||
s3grg put /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.s3cmd
|
||||
s3grg put /tmp/garage.$idx s3://eprouvette/garage.$idx.s3cmd
|
||||
|
||||
s3grg ls s3://eprouvette
|
||||
|
||||
s3grg get s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
awsgrg cp s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
s3grg rm s3://eprouvette/garage.$idx.s3cmd
|
||||
done
|
||||
rm /tmp/garage.{1,2,3}.rnd
|
||||
rm /tmp/garage.{1,2,3}.{rnd,b64}
|
||||
|
||||
echo "website testing"
|
||||
echo "<h1>hello world</h1>" > /tmp/garage-index.html
|
||||
|
|
|
|||
|
|
@ -22,10 +22,12 @@ bytes = "1.0"
|
|||
chrono = "0.4"
|
||||
crypto-mac = "0.10"
|
||||
err-derive = "0.3"
|
||||
fastcdc = "1.0.5"
|
||||
hex = "0.4"
|
||||
hmac = "0.10"
|
||||
log = "0.4"
|
||||
md-5 = "0.9"
|
||||
rand = "0.7"
|
||||
sha2 = "0.9"
|
||||
|
||||
futures = "0.3"
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ use crate::s3_get::*;
|
|||
use crate::s3_list::*;
|
||||
use crate::s3_put::*;
|
||||
|
||||
/// Run the S3 API server
|
||||
pub async fn run_api_server(
|
||||
garage: Arc<Garage>,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
//! Module containing various helpers for encoding
|
||||
|
||||
/// Escape &str for xml inclusion
|
||||
pub fn xml_escape(s: &str) -> String {
|
||||
s.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace("\"", """)
|
||||
}
|
||||
|
||||
/// Encode &str for use in a URI
|
||||
pub fn uri_encode(string: &str, encode_slash: bool) -> String {
|
||||
let mut result = String::with_capacity(string.len() * 2);
|
||||
for c in string.chars() {
|
||||
|
|
@ -24,6 +28,7 @@ pub fn uri_encode(string: &str, encode_slash: bool) -> String {
|
|||
result
|
||||
}
|
||||
|
||||
/// Encode &str either as an uri, or a valid string for xml inclusion
|
||||
pub fn xml_encode_key(k: &str, urlencode: bool) -> String {
|
||||
if urlencode {
|
||||
uri_encode(k, true)
|
||||
|
|
|
|||
|
|
@ -3,44 +3,57 @@ use hyper::StatusCode;
|
|||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
// Category: internal error
|
||||
/// Error related to deeper parts of Garage
|
||||
#[error(display = "Internal error: {}", _0)]
|
||||
InternalError(#[error(source)] GarageError),
|
||||
|
||||
/// Error related to Hyper
|
||||
#[error(display = "Internal error (Hyper error): {}", _0)]
|
||||
Hyper(#[error(source)] hyper::Error),
|
||||
|
||||
/// Error related to HTTP
|
||||
#[error(display = "Internal error (HTTP error): {}", _0)]
|
||||
HTTP(#[error(source)] http::Error),
|
||||
|
||||
// Category: cannot process
|
||||
/// No proper api key was used, or the signature was invalid
|
||||
#[error(display = "Forbidden: {}", _0)]
|
||||
Forbidden(String),
|
||||
|
||||
/// The object requested don't exists
|
||||
#[error(display = "Not found")]
|
||||
NotFound,
|
||||
|
||||
// Category: bad request
|
||||
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUTF8Str(#[error(source)] std::str::Utf8Error),
|
||||
|
||||
/// The request used an invalid path
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUTF8String(#[error(source)] std::string::FromUtf8Error),
|
||||
|
||||
/// Some base64 encoded data was badly encoded
|
||||
#[error(display = "Invalid base64: {}", _0)]
|
||||
InvalidBase64(#[error(source)] base64::DecodeError),
|
||||
|
||||
/// The client sent invalid XML data
|
||||
#[error(display = "Invalid XML: {}", _0)]
|
||||
InvalidXML(String),
|
||||
|
||||
/// The client sent a header with invalid value
|
||||
#[error(display = "Invalid header value: {}", _0)]
|
||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
||||
|
||||
/// The client sent a range header with invalid value
|
||||
#[error(display = "Invalid HTTP range: {:?}", _0)]
|
||||
InvalidRange(#[error(from)] http_range::HttpRangeParseError),
|
||||
|
||||
/// The client sent an invalid request
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
}
|
||||
|
|
@ -52,6 +65,7 @@ impl From<roxmltree::Error> for Error {
|
|||
}
|
||||
|
||||
impl Error {
|
||||
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||
pub fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::NotFound => StatusCode::NOT_FOUND,
|
||||
|
|
@ -65,6 +79,7 @@ impl Error {
|
|||
}
|
||||
}
|
||||
|
||||
/// Trait to map error to the Bad Request error code
|
||||
pub trait OkOrBadRequest {
|
||||
type S2;
|
||||
fn ok_or_bad_request(self, reason: &'static str) -> Self::S2;
|
||||
|
|
@ -93,6 +108,7 @@ impl<T> OkOrBadRequest for Option<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Trait to map an error to an Internal Error code
|
||||
pub trait OkOrInternalError {
|
||||
type S2;
|
||||
fn ok_or_internal_error(self, reason: &'static str) -> Self::S2;
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue