Compare commits

..
Author SHA1 Message Date
Quentin Dufour
1ebaf7aa17
force flag "no read ahead" on LMDB 2024-08-17 13:17:16 +02:00
Quentin Dufour
306a74379a
add metrics to workers 2024-08-17 13:16:55 +02:00
Quentin Dufour
14163b5853
switch to ms, simplify collected metrics 2024-08-15 15:43:15 +02:00
Quentin Dufour
2d439c388c
switch from micros to millis 2024-08-15 15:30:56 +02:00
Quentin Dufour
1685d83c04
switch from sec to us 2024-08-15 14:32:01 +02:00
Quentin Dufour
9d01a9870c
fix cargo.nix 2024-08-15 09:10:39 +02:00
Quentin Dufour
56de00945f
proxy finalized, use it by setting 'lmdb-with-metrics' 2024-08-14 23:12:46 +02:00
Quentin Dufour
efc87a8b8e
add proxy to instrument LmdbDB with otel 2024-08-14 22:20:08 +02:00
197 changed files with 11437 additions and 7701 deletions

3
.cargo/config.toml Normal file
View file

@ -0,0 +1,3 @@
[target.x86_64-unknown-linux-gnu]
linker = "clang"
rustflags = ["-C", "link-arg=-fuse-ld=mold"]

View file

@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- push
@ -12,33 +9,39 @@ when:
steps:
- name: check formatting
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.fmt
- nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.dev
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- name: unit + func tests (lmdb)
image: nixpkgs/nix:nixos-24.05
- name: unit + func tests
image: nixpkgs/nix:nixos-22.05
environment:
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
commands:
- nix-build -j4 --attr flakePackages.tests-lmdb
- name: unit + func tests (sqlite)
image: nixpkgs/nix:nixos-24.05
commands:
- nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-24.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-build --no-build-output --attr test.amd64
- ./result/bin/garage_db-*
- ./result/bin/garage_api-*
- ./result/bin/garage_model-*
- ./result/bin/garage_rpc-*
- ./result/bin/garage_table-*
- ./result/bin/garage_util-*
- ./result/bin/garage_web-*
- ./result/bin/garage-*
- GARAGE_TEST_INTEGRATION_DB_ENGINE=lmdb ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- nix-shell --attr ci --run "killall -9 garage" || true
- GARAGE_TEST_INTEGRATION_DB_ENGINE=sqlite ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- rm result
- rm -rv tmp-garage-integration
- name: integration tests
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.dev
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
depends_on: [ build ]

View file

@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
@ -11,21 +8,20 @@ depends_on:
steps:
- name: refresh-index
image: nixpkgs/nix:nixos-24.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
image: nixpkgs/nix:nixos-22.05
secrets:
- source: garagehq_aws_access_key_id
target: AWS_ACCESS_KEY_ID
- source: garagehq_aws_secret_access_key
target: AWS_SECRET_ACCESS_KEY
commands:
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- nix-shell --attr ci --run "refresh_index"
- name: multiarch-docker
image: nixpkgs/nix:nixos-24.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
image: nixpkgs/nix:nixos-22.05
secrets:
- docker_auth
commands:
- mkdir -p /root/.docker
- echo $DOCKER_AUTH > /root/.docker/config.json

View file

@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
@ -19,17 +16,18 @@ matrix:
steps:
- name: build
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- name: check is static binary
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
- nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
when:
@ -39,7 +37,7 @@ steps:
ARCH: i386
- name: upgrade tests
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
when:
@ -47,23 +45,24 @@ steps:
ARCH: amd64
- name: push static binary
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
TARGET: "${TARGET}"
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
secrets:
- source: garagehq_aws_access_key_id
target: AWS_ACCESS_KEY_ID
- source: garagehq_aws_secret_access_key
target: AWS_SECRET_ACCESS_KEY
commands:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_PLATFORM: "linux/${ARCH}"
CONTAINER_NAME: "dxflrs/${ARCH}_garage"
DOCKER_AUTH:
from_secret: docker_auth
secrets:
- docker_auth
commands:
- mkdir -p /root/.docker
- echo $DOCKER_AUTH > /root/.docker/config.json

2941
Cargo.lock generated

File diff suppressed because it is too large Load diff

7102
Cargo.nix Normal file

File diff suppressed because it is too large Load diff

View file

@ -8,10 +8,7 @@ members = [
"src/table",
"src/block",
"src/model",
"src/api/common",
"src/api/s3",
"src/api/k2v",
"src/api/admin",
"src/api",
"src/web",
"src/garage",
"src/k2v-client",
@ -24,18 +21,15 @@ default-members = ["src/garage"]
# Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "1.3.1", path = "src/api/common" }
garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
garage_block = { version = "1.3.1", path = "src/block" }
garage_db = { version = "1.3.1", path = "src/db", default-features = false }
garage_model = { version = "1.3.1", path = "src/model", default-features = false }
garage_net = { version = "1.3.1", path = "src/net" }
garage_rpc = { version = "1.3.1", path = "src/rpc" }
garage_table = { version = "1.3.1", path = "src/table" }
garage_util = { version = "1.3.1", path = "src/util" }
garage_web = { version = "1.3.1", path = "src/web" }
garage_api = { version = "1.0.0", path = "src/api" }
garage_block = { version = "1.0.0", path = "src/block" }
garage_db = { version = "1.0.0", path = "src/db", default-features = false }
garage_model = { version = "1.0.0", path = "src/model", default-features = false }
garage_net = { version = "1.0.0", path = "src/net" }
garage_rpc = { version = "1.0.0", path = "src/rpc" }
garage_table = { version = "1.0.0", path = "src/table" }
garage_util = { version = "1.0.0", path = "src/util" }
garage_web = { version = "1.0.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io
@ -52,20 +46,21 @@ chrono = "0.4"
crc32fast = "1.4"
crc32c = "0.6"
crypto-common = "0.1"
digest = "0.10"
err-derive = "0.3"
gethostname = "0.4"
git-version = "0.3.4"
hex = "0.4"
hexdump = "0.1"
html-escape = "0.2.13"
hmac = "0.12"
idna = "0.5"
itertools = "0.12"
ipnet = "2.9.0"
lazy_static = "1.4"
md-5 = "0.10"
mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] }
nix = { version = "0.27", default-features = false, features = ["fs"] }
nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1"
pin-project = "1.0.12"
pnet_datalink = "0.34"
@ -84,14 +79,12 @@ pretty_env_logger = "0.5"
structopt = { version = "0.3", default-features = false }
syslog-tracing = "0.3"
tracing = "0.1"
tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.37"
rusqlite = "0.31.0"
r2d2 = "0.8"
r2d2_sqlite = "0.31"
fjall = "2.4"
r2d2_sqlite = "0.24"
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false }
@ -134,21 +127,26 @@ opentelemetry-contrib = "0.9"
prometheus = "0.13"
# used by the k2v-client crate only
aws-sigv4 = { version = "1.1", default-features = false }
hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
aws-sigv4 = { version = "1.1" }
hyper-rustls = { version = "0.26", features = ["http2"] }
log = "0.4"
thiserror = "2.0"
thiserror = "1.0"
# ---- used only as build / dev dependencies ----
assert-json-diff = "2.0"
rustc_version = "0.4.0"
static_init = "1.0"
aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
aws-sdk-config = { version = "1.62", default-features = false }
aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
aws-config = "1.1.4"
aws-sdk-config = "1.13"
aws-sdk-s3 = "1.14"
[profile.dev]
#lto = "thin" # disabled for now, adds 2-4 min to each CI build
lto = "off"
[profile.release]
lto = "thin"
codegen-units = 16
opt-level = 3
strip = "debuginfo"
lto = true
codegen-units = 1
opt-level = "s"
strip = true

View file

@ -3,5 +3,5 @@ FROM scratch
ENV RUST_BACKTRACE=1
ENV RUST_LOG=garage=info
COPY result/bin/garage /
COPY result-bin/bin/garage /
CMD [ "/garage", "server"]

View file

@ -1,8 +1,13 @@
.PHONY: doc all run1 run2 run3
.PHONY: doc all release shell run1 run2 run3
all:
clear
cargo build
clear; cargo build
release:
nix-build --attr pkgs.amd64.release --no-build-output
shell:
nix-shell
# ----

View file

@ -3,22 +3,53 @@
with import ./nix/common.nix;
let
pkgs = import nixpkgs { };
pkgs = import pkgsSrc { };
compile = import ./nix/compile.nix;
build_release = target: (compile {
inherit target system git_version nixpkgs;
crane = flake.inputs.crane;
rust-overlay = flake.inputs.rust-overlay;
release = true;
}).garage;
build_debug_and_release = (target: {
debug = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = false;
}).workspace.garage { compileMode = "build"; };
release = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = true;
}).workspace.garage { compileMode = "build"; };
});
test = (rustPkgs:
pkgs.symlinkJoin {
name = "garage-tests";
paths =
builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
(builtins.attrNames rustPkgs.workspace);
});
in {
releasePackages = {
amd64 = build_release "x86_64-unknown-linux-musl";
i386 = build_release "i686-unknown-linux-musl";
arm64 = build_release "aarch64-unknown-linux-musl";
arm = build_release "armv6l-unknown-linux-musleabihf";
pkgs = {
amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
i386 = build_debug_and_release "i686-unknown-linux-musl";
arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
};
test = {
amd64 = test (compile {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
features = [
"garage/bundled-libs"
"garage/k2v"
"garage/lmdb"
"garage/sqlite"
];
});
};
clippy = {
amd64 = (compile {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
compiler = "clippy";
}).workspace.garage { compileMode = "build"; };
};
flakePackages = flake.packages.${system};
}

View file

@ -687,7 +687,7 @@ paths:
operationId: "GetBucketInfo"
summary: "Get a bucket"
description: |
Given a bucket identifier (`id`) or a global alias (`globalAlias`), get its information.
Given a bucket identifier (`id`) or a global alias (`alias`), get its information.
It includes its aliases, its web configuration, keys that have some permissions
on it, some statistics (number of objects, size), number of dangling multipart uploads,
and its quotas (if any).
@ -701,7 +701,7 @@ paths:
example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87"
schema:
type: string
- name: globalAlias
- name: alias
in: query
description: |
The exact global alias of one of the existing buckets.

View file

@ -23,7 +23,7 @@ client = minio.Minio(
"GKyourapikey",
"abcd[...]1234",
# Force the region, this is specific to garage
region="garage",
region="region",
)
```

View file

@ -12,7 +12,7 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | ✅ | Natively supported |
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
| [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
@ -69,7 +69,7 @@ $CONFIG = array(
'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
'port' => 3900, // Put your reverse proxy port or your S3 API port
'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy
'region' => 'garage', // Garage default region is named "garage", edit according to your cluster config
'region' => 'garage', // Garage has only one region named "garage"
'use_path_style' => true // Garage supports only path style, must be set to true
],
],
@ -135,7 +135,7 @@ bucket but doesn't also know the secret encryption key.
*Click on the picture to zoom*
Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
@ -191,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
```bash
garage bucket create peertube-videos
garage bucket create peertube-playlists
garage bucket create peertube-playlist
```
Now we allow our key to read and write on these buckets:
@ -238,7 +238,7 @@ object_storage:
# Put localhost only if you have a garage instance running on that node
endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
# Garage default region is named "garage", edit according to your config
# Garage supports only one region for now, named garage
region: 'garage'
credentials:
@ -253,7 +253,7 @@ object_storage:
proxify_private_files: false
streaming_playlists:
bucket_name: 'peertube-playlists'
bucket_name: 'peertube-playlist'
# Keep it empty for our example
prefix: ''
@ -335,7 +335,6 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t
```bash
$ RAILS_ENV=production bin/tootctl media remove --days 3
$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles
$ RAILS_ENV=production bin/tootctl media remove-orphans
$ RAILS_ENV=production bin/tootctl preview_cards remove --days 15
```
@ -354,6 +353,8 @@ Imports: 1.7 KB
Settings: 0 Bytes
```
Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567).
### Migrating your data
Data migration should be done with an efficient S3 client.
@ -441,7 +442,7 @@ media_storage_providers:
store_synchronous: True # do we want to wait that the file has been written before returning?
config:
bucket: matrix # the name of our bucket, we chose matrix earlier
region_name: garage # "garage" by default, edit according to your cluster config
region_name: garage # only "garage" is supported for the region field
endpoint_url: http://localhost:3900 # the path to the S3 endpoint
access_key_id: "GKxxx" # your Key ID
secret_access_key: "xxxx" # your Secret Key

View file

@ -161,49 +161,3 @@ kopia repository validate-provider
You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
Everything should work out-of-the-box.
## Plakar
Create your key and bucket on Garage server:
```bash
garage key create my-plakar-key
garage bucket create plakar-backups
garage bucket allow plakar-backups --read --write --key my-plakar-key
…
```
On Plakar server, add your Garage as a storage location:
```bash
plakar store add garageS3 s3://my-garage.tld/plakar-backups \
region=garage # Or as you've specified in garage.toml \
access_key=<Key ID from "garage key info my-plakar-key"> \
secret_access_key=<Secret key from "garage key info my-plakar-key">
```
Then create the repository.
```bash
plakar at @garageS3 create -plaintext # Unencrypted
# or
plakar at @garageS3 create #encrypted
```
If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
```bash
plakar at @garageS3 check
```
Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
```bash
$ plakar at ~/backups sync to @garageS3
```
Or list the S3 storage content:
```bash
$ plakar at @garageS3 ls
```
More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/

View file

@ -17,7 +17,7 @@ Garage can also help you serve this content.
## Gitea
You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachments.
You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachements.
You can configure a different target for each data type (check `[lfs]` and `[attachment]` sections of the Gitea documentation) and you can provide a default one through the `[storage]` section.
Let's start by creating a key and a bucket (your key id and secret will be needed later, keep them somewhere):

View file

@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below.
## Comparison of Ansible roles
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
| **Runtime** | Systemd | Docker | Systemd |
| **Target OS** | Any Linux | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 |
| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) |
| **Automatic node connection** | ❌ | ✅ | ✅ |
| **Layout management** | ❌ | ✅ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ |
| **Allow custom Garage config** | ✅ | ❌ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ |
| **Multiple instances on one host** | ✅ | ✅ | ❌ |
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
| **Runtime** | Systemd | Docker |
| **Target OS** | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 |
| **Additional software** | None | Traefik |
| **Automatic node connection** | ❌ | ✅ |
| **Layout management** | ❌ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) |
| **Allow custom Garage config** | ✅ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ |
| **Multiple instances on one host** | ✅ | ✅ |
## zorun/ansible-role-garage
@ -49,15 +49,3 @@ structured DNS names, etc).
As a result, this role makes it easier to start with Garage on Ansible,
but is less flexible.
## eddster2309/ansible-role-garage
[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
This role is a opinionated but customisable role using the official Garage
static binaries and only requires Systemd. As such it should work on any
Linux based host. It includes all the nesscary configuration to
automatically setup a clustered Garage deployment. Most Garage
configuration options are exposed through Ansible variables so while you
can't provide a custom config you can get very close. It can optionally
installed a HA nginx deployment with Keepalived.

View file

@ -15,10 +15,9 @@ Alpine Linux repositories (available since v3.17):
apk add garage
```
The default configuration file is installed to `/etc/garage/garage.toml`. You can run
Garage using: `rc-service garage start`.
If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
The default configuration file is installed to `/etc/garage.toml`. You can run
Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
will be automatically replaced with a random string on the first start.
Please note that this package is built without Consul discovery, Kubernetes
discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@ -27,7 +26,7 @@ it's stable).
## Arch Linux
Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
## FreeBSD

View file

@ -11,7 +11,7 @@ Firstly clone the repository:
```bash
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
cd garage/script/helm
cd garage/scripts/helm
```
Deploy with default options:
@ -26,13 +26,6 @@ Or deploy with custom values:
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart:
```bash
kubectl apply -k ../k8s/crd
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
```bash
@ -93,62 +86,3 @@ helm delete --namespace garage garage
```
Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
## Increase PVC size on running Garage instances
Since the Garage Helm chart creates the data and meta PVC based on `StatefulSet` templates, increasing the PVC size can be a bit tricky.
### Confirm the `StorageClass` used for Garage supports volume expansion
Confirm the storage class used for garage.
```bash
kubectl -n garage get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
data-garage-0 Bound pvc-080360c9-8ce3-4acf-8579-1701e57b7f3f 30Gi RWO longhorn-local <unset> 77d
data-garage-1 Bound pvc-ab8ba697-6030-4fc7-ab3c-0d6df9e3dbc0 30Gi RWO longhorn-local <unset> 5d8h
data-garage-2 Bound pvc-3ab37551-0231-4604-986d-136d0fd950ec 30Gi RWO longhorn-local <unset> 5d5h
meta-garage-0 Bound pvc-3b457302-3023-4169-846e-c928c5f2ea65 3Gi RWO longhorn-local <unset> 77d
meta-garage-1 Bound pvc-49ace2b9-5c85-42df-9247-51c4cf64b460 3Gi RWO longhorn-local <unset> 5d8h
meta-garage-2 Bound pvc-99e2e50f-42b4-4128-ae2f-b52629259723 3Gi RWO longhorn-local <unset> 5d5h
```
In this case, the storage class is `longhorn-local`. Now, check if `ALLOWVOLUMEEXPANSION` is true for the used `StorageClass`.
```bash
kubectl get storageclasses.storage.k8s.io longhorn-local
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
longhorn-local driver.longhorn.io Delete Immediate true 103d
```
If your `StorageClass` does not support volume expansion, double check if you can enable it. Otherwise, your only real option is to spin up a new Garage cluster with increased size and migrate all data over.
If your `StorageClass` supports expansion, you are free to continue.
### Increase the size of the PVCs
Increase the size of all PVCs to your desired size.
```bash
kubectl -n garage edit pvc data-garage-0
kubectl -n garage edit pvc data-garage-1
kubectl -n garage edit pvc data-garage-2
kubectl -n garage edit pvc meta-garage-0
kubectl -n garage edit pvc meta-garage-1
kubectl -n garage edit pvc meta-garage-2
```
### Increase the size of the `StatefulSet` PVC template
This is an optional step, but if not done, future instances of Garage will be created with the original size from the template.
```bash
kubectl -n garage delete sts --cascade=orphan garage
statefulset.apps "garage" deleted
```
This will remove the Garage `StatefulSet` but leave the pods running. It may seem destructive but needs to be done this way since edits to the size of PVC templates are prohibited.
### Redeploy the `StatefulSet`
Now the size of future PVCs can be increased, and the Garage Helm chart can be upgraded. The new `StatefulSet` should take ownership of the orphaned pods again.

View file

@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
We encourage you to use a fixed tag (eg. `v1.0.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.0.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
sudo docker pull dxflrs/garage:v1.3.0
sudo docker pull dxflrs/garage:v1.0.0
```
## Deploying and configuring Garage
@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.3.0
dxflrs/garage:v1.0.0
```
With this command line, Garage should be started automatically at each boot.
@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v1.3.0
image: dxflrs/garage:v1.0.0
network_mode: "host"
restart: unless-stopped
volumes:

View file

@ -28,7 +28,6 @@ StateDirectory=garage
DynamicUser=true
ProtectHome=true
NoNewPrivileges=true
LimitNOFILE=42000
[Install]
WantedBy=multi-user.target

Some files were not shown because too many files have changed in this diff Show more