mirror of
https://git.deuxfleurs.fr/Deuxfleurs/garage.git
synced 2026-11-09 15:49:16 -05:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
b1685514d0 |
500 changed files with 19968 additions and 87797 deletions
288
.drone.yml
Normal file
288
.drone.yml
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
---
|
||||
kind: pipeline
|
||||
name: default
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: check formatting
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr rust --run "cargo fmt -- --check"
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
|
||||
- name: unit + func tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr test.amd64
|
||||
- ./result/bin/garage_db-*
|
||||
- ./result/bin/garage_api-*
|
||||
- ./result/bin/garage_model-*
|
||||
- ./result/bin/garage_rpc-*
|
||||
- ./result/bin/garage_table-*
|
||||
- ./result/bin/garage_util-*
|
||||
- ./result/bin/garage_web-*
|
||||
- ./result/bin/garage-*
|
||||
- ./result/bin/integration-*
|
||||
- rm result
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- custom
|
||||
- push
|
||||
- pull_request
|
||||
- tag
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-amd64
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: integration
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "x86_64-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/amd64"
|
||||
CONTAINER_NAME: "dxflrs/amd64_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-i386
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: integration
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "i686-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/386"
|
||||
CONTAINER_NAME: "dxflrs/386_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-arm64
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "aarch64-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/arm64"
|
||||
CONTAINER_NAME: "dxflrs/arm64_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-arm
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "armv6l-unknown-linux-musleabihf"
|
||||
commands:
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/arm"
|
||||
CONTAINER_NAME: "dxflrs/arm_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: refresh-release-page
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: multiarch-docker
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
HOME: "/root"
|
||||
commands:
|
||||
- mkdir -p /root/.docker
|
||||
- echo $DOCKER_AUTH > /root/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr release --run "multiarch_docker"
|
||||
- name: refresh-index
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-shell --attr release --run "refresh_index"
|
||||
|
||||
depends_on:
|
||||
- release-linux-amd64
|
||||
- release-linux-i386
|
||||
- release-linux-arm64
|
||||
- release-linux-arm
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: signature
|
||||
hmac: ac09a5a8c82502f67271f93afa1e1e21ce66383b8e24a6deb26b285cc1c378ba
|
||||
|
||||
...
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
labels:
|
||||
nix: "enabled"
|
||||
|
||||
when:
|
||||
event:
|
||||
- push
|
||||
- tag
|
||||
- pull_request
|
||||
- deployment
|
||||
- cron
|
||||
- manual
|
||||
|
||||
steps:
|
||||
- name: check formatting
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.fmt
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.dev
|
||||
|
||||
- name: unit + func tests (lmdb)
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.tests-lmdb
|
||||
|
||||
- name: unit + func tests (sqlite)
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.tests-sqlite
|
||||
|
||||
- name: unit + func tests (fjall)
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.tests-fjall
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build -j4 --attr flakePackages.dev
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
depends_on: [ build ]
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
labels:
|
||||
nix: "enabled"
|
||||
|
||||
when:
|
||||
event:
|
||||
- deployment
|
||||
- cron
|
||||
|
||||
depends_on:
|
||||
- release
|
||||
|
||||
steps:
|
||||
- name: refresh-index
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-shell --attr ci --run "refresh_index"
|
||||
|
||||
- name: multiarch-docker
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
commands:
|
||||
- mkdir -p /root/.docker
|
||||
- echo $DOCKER_AUTH > /root/.docker/config.json
|
||||
- export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-shell --attr ci --run "multiarch_docker"
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
labels:
|
||||
nix: "enabled"
|
||||
|
||||
when:
|
||||
event:
|
||||
- deployment
|
||||
- cron
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- ARCH: amd64
|
||||
TARGET: x86_64-unknown-linux-musl
|
||||
- ARCH: i386
|
||||
TARGET: i686-unknown-linux-musl
|
||||
- ARCH: arm64
|
||||
TARGET: aarch64-unknown-linux-musl
|
||||
- ARCH: arm
|
||||
TARGET: armv6l-unknown-linux-musleabihf
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
|
||||
- name: check is static binary
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
when:
|
||||
- matrix:
|
||||
ARCH: amd64
|
||||
- matrix:
|
||||
ARCH: i386
|
||||
|
||||
- name: upgrade tests
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
|
||||
when:
|
||||
- matrix:
|
||||
ARCH: amd64
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
environment:
|
||||
TARGET: "${TARGET}"
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-24.05
|
||||
environment:
|
||||
DOCKER_PLATFORM: "linux/${ARCH}"
|
||||
CONTAINER_NAME: "dxflrs/${ARCH}_garage"
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
commands:
|
||||
- mkdir -p /root/.docker
|
||||
- echo $DOCKER_AUTH > /root/.docker/config.json
|
||||
- export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-shell --attr ci --run "to_docker"
|
||||
4219
Cargo.lock
generated
4219
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
140
Cargo.toml
140
Cargo.toml
|
|
@ -3,152 +3,20 @@ resolver = "2"
|
|||
members = [
|
||||
"src/db",
|
||||
"src/util",
|
||||
"src/net",
|
||||
"src/rpc",
|
||||
"src/table",
|
||||
"src/block",
|
||||
"src/model",
|
||||
"src/api/common",
|
||||
"src/api/s3",
|
||||
"src/api/k2v",
|
||||
"src/api/admin",
|
||||
"src/api",
|
||||
"src/web",
|
||||
"src/garage",
|
||||
"src/k2v-client",
|
||||
"src/format-table",
|
||||
]
|
||||
|
||||
default-members = ["src/garage"]
|
||||
|
||||
[workspace.dependencies]
|
||||
|
||||
# Internal Garage crates
|
||||
format_table = { version = "0.1.1", path = "src/format-table" }
|
||||
garage_api_common = { version = "1.3.1", path = "src/api/common" }
|
||||
garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
|
||||
garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
|
||||
garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
|
||||
garage_block = { version = "1.3.1", path = "src/block" }
|
||||
garage_db = { version = "1.3.1", path = "src/db", default-features = false }
|
||||
garage_model = { version = "1.3.1", path = "src/model", default-features = false }
|
||||
garage_net = { version = "1.3.1", path = "src/net" }
|
||||
garage_rpc = { version = "1.3.1", path = "src/rpc" }
|
||||
garage_table = { version = "1.3.1", path = "src/table" }
|
||||
garage_util = { version = "1.3.1", path = "src/util" }
|
||||
garage_web = { version = "1.3.1", path = "src/web" }
|
||||
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
|
||||
|
||||
# External crates from crates.io
|
||||
arc-swap = "1.0"
|
||||
argon2 = "0.5"
|
||||
async-trait = "0.1.7"
|
||||
backtrace = "0.3"
|
||||
base64 = "0.21"
|
||||
blake2 = "0.10"
|
||||
bytes = "1.0"
|
||||
bytesize = "1.1"
|
||||
cfg-if = "1.0"
|
||||
chrono = "0.4"
|
||||
crc32fast = "1.4"
|
||||
crc32c = "0.6"
|
||||
crypto-common = "0.1"
|
||||
gethostname = "0.4"
|
||||
git-version = "0.3.4"
|
||||
hex = "0.4"
|
||||
hexdump = "0.1"
|
||||
html-escape = "0.2.13"
|
||||
hmac = "0.12"
|
||||
itertools = "0.12"
|
||||
ipnet = "2.9.0"
|
||||
lazy_static = "1.4"
|
||||
md-5 = "0.10"
|
||||
mktemp = "0.5"
|
||||
nix = { version = "0.29", default-features = false, features = ["fs"] }
|
||||
nom = "7.1"
|
||||
parking_lot = "0.12"
|
||||
parse_duration = "2.1"
|
||||
pin-project = "1.0.12"
|
||||
pnet_datalink = "0.34"
|
||||
rand = "0.8"
|
||||
sha1 = "0.10"
|
||||
sha2 = "0.10"
|
||||
timeago = { version = "0.4", default-features = false }
|
||||
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
||||
|
||||
aes-gcm = { version = "0.10", features = ["aes", "stream"] }
|
||||
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
||||
kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
|
||||
|
||||
clap = { version = "4.1", features = ["derive", "env"] }
|
||||
pretty_env_logger = "0.5"
|
||||
structopt = { version = "0.3", default-features = false }
|
||||
syslog-tracing = "0.3"
|
||||
tracing = "0.1"
|
||||
tracing-journald = "0.3.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
|
||||
rusqlite = "0.37"
|
||||
r2d2 = "0.8"
|
||||
r2d2_sqlite = "0.31"
|
||||
fjall = "2.4"
|
||||
|
||||
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
|
||||
zstd = { version = "0.13", default-features = false }
|
||||
|
||||
quick-xml = { version = "0.26", features = [ "serialize" ] }
|
||||
rmp-serde = "1.1.2"
|
||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||
serde_bytes = "0.11"
|
||||
serde_json = "1.0"
|
||||
toml = { version = "0.8", default-features = false, features = ["parse"] }
|
||||
|
||||
# newer version requires rust edition 2021
|
||||
k8s-openapi = { version = "0.21", features = ["v1_24"] }
|
||||
kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] }
|
||||
schemars = "0.8"
|
||||
reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] }
|
||||
|
||||
form_urlencoded = "1.0.0"
|
||||
http = "1.0"
|
||||
httpdate = "1.0"
|
||||
http-range = "0.1"
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1.0", default-features = false }
|
||||
hyper-util = { version = "0.1", features = [ "full" ] }
|
||||
multer = "3.0"
|
||||
percent-encoding = "2.2"
|
||||
roxmltree = "0.19"
|
||||
url = "2.3"
|
||||
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-util = { version = "0.7", features = ["compat", "io"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
opentelemetry-contrib = "0.9"
|
||||
prometheus = "0.13"
|
||||
|
||||
# used by the k2v-client crate only
|
||||
aws-sigv4 = { version = "1.1", default-features = false }
|
||||
hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
|
||||
log = "0.4"
|
||||
thiserror = "2.0"
|
||||
|
||||
# ---- used only as build / dev dependencies ----
|
||||
assert-json-diff = "2.0"
|
||||
rustc_version = "0.4.0"
|
||||
static_init = "1.0"
|
||||
aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
|
||||
aws-sdk-config = { version = "1.62", default-features = false }
|
||||
aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
|
||||
[profile.dev]
|
||||
lto = "off"
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 16
|
||||
opt-level = 3
|
||||
strip = "debuginfo"
|
||||
debug = true
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@ FROM scratch
|
|||
ENV RUST_BACKTRACE=1
|
||||
ENV RUST_LOG=garage=info
|
||||
|
||||
COPY result/bin/garage /
|
||||
COPY result-bin/bin/garage /
|
||||
CMD [ "/garage", "server"]
|
||||
|
|
|
|||
11
Makefile
11
Makefile
|
|
@ -1,8 +1,13 @@
|
|||
.PHONY: doc all run1 run2 run3
|
||||
.PHONY: doc all release shell run1 run2 run3
|
||||
|
||||
all:
|
||||
clear
|
||||
cargo build
|
||||
clear; cargo build
|
||||
|
||||
release:
|
||||
nix-build --attr pkgs.amd64.release --no-build-output
|
||||
|
||||
shell:
|
||||
nix-shell
|
||||
|
||||
# ----
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Garage [](https://woodpecker.deuxfleurs.fr/repos/1)
|
||||
Garage [](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
|
||||
===
|
||||
|
||||
<p align="center" style="text-align:center;">
|
||||
|
|
|
|||
58
default.nix
58
default.nix
|
|
@ -3,22 +3,54 @@
|
|||
with import ./nix/common.nix;
|
||||
|
||||
let
|
||||
pkgs = import nixpkgs { };
|
||||
pkgs = import pkgsSrc { };
|
||||
compile = import ./nix/compile.nix;
|
||||
|
||||
build_release = target: (compile {
|
||||
inherit target system git_version nixpkgs;
|
||||
crane = flake.inputs.crane;
|
||||
rust-overlay = flake.inputs.rust-overlay;
|
||||
release = true;
|
||||
}).garage;
|
||||
build_debug_and_release = (target: {
|
||||
debug = (compile {
|
||||
inherit system target git_version pkgsSrc cargo2nixOverlay;
|
||||
release = false;
|
||||
}).workspace.garage { compileMode = "build"; };
|
||||
|
||||
release = (compile {
|
||||
inherit system target git_version pkgsSrc cargo2nixOverlay;
|
||||
release = true;
|
||||
}).workspace.garage { compileMode = "build"; };
|
||||
});
|
||||
|
||||
test = (rustPkgs:
|
||||
pkgs.symlinkJoin {
|
||||
name = "garage-tests";
|
||||
paths =
|
||||
builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
|
||||
(builtins.attrNames rustPkgs.workspace);
|
||||
});
|
||||
|
||||
in {
|
||||
releasePackages = {
|
||||
amd64 = build_release "x86_64-unknown-linux-musl";
|
||||
i386 = build_release "i686-unknown-linux-musl";
|
||||
arm64 = build_release "aarch64-unknown-linux-musl";
|
||||
arm = build_release "armv6l-unknown-linux-musleabihf";
|
||||
pkgs = {
|
||||
amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
|
||||
i386 = build_debug_and_release "i686-unknown-linux-musl";
|
||||
arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
|
||||
arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
|
||||
};
|
||||
test = {
|
||||
amd64 = test (compile {
|
||||
inherit system git_version pkgsSrc cargo2nixOverlay;
|
||||
target = "x86_64-unknown-linux-musl";
|
||||
features = [
|
||||
"garage/bundled-libs"
|
||||
"garage/k2v"
|
||||
"garage/sled"
|
||||
"garage/lmdb"
|
||||
"garage/sqlite"
|
||||
];
|
||||
});
|
||||
};
|
||||
clippy = {
|
||||
amd64 = (compile {
|
||||
inherit system git_version pkgsSrc cargo2nixOverlay;
|
||||
target = "x86_64-unknown-linux-musl";
|
||||
compiler = "clippy";
|
||||
}).workspace.garage { compileMode = "build"; };
|
||||
};
|
||||
flakePackages = flake.packages.${system};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -632,7 +632,7 @@ paths:
|
|||
operationId: "UpdateBucket"
|
||||
summary: "Update a bucket"
|
||||
description: |
|
||||
All fields (`websiteAccess` and `quotas`) are optional.
|
||||
All fields (`websiteAccess` and `quotas`) are optionnal.
|
||||
If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
|
||||
|
||||
In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Garage Adminstration API v0</title>
|
||||
<!-- needed for adaptive design -->
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="./css/redoc.css" rel="stylesheet">
|
||||
|
||||
<!--
|
||||
Redoc doesn't change outer page styles
|
||||
-->
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<redoc spec-url='./garage-admin-v1.yml'></redoc>
|
||||
<script src="./redoc.standalone.js"> </script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Build your own app"
|
||||
weight = 40
|
||||
weight = 4
|
||||
sort_by = "weight"
|
||||
template = "documentation.html"
|
||||
+++
|
||||
|
|
|
|||
|
|
@ -37,84 +37,30 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Initialization
|
||||
// Set Host and other parameters
|
||||
configuration := garage.NewConfiguration()
|
||||
configuration.Host = "127.0.0.1:3903"
|
||||
|
||||
|
||||
// We can now generate a client
|
||||
client := garage.NewAPIClient(configuration)
|
||||
|
||||
// Authentication is handled through the context pattern
|
||||
ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t")
|
||||
|
||||
// Nodes
|
||||
fmt.Println("--- nodes ---")
|
||||
nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute()
|
||||
fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname)
|
||||
capa := int64(1000000000)
|
||||
change := []garage.NodeRoleChange{
|
||||
garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate {
|
||||
Id: *nodes.KnownNodes[0].Id,
|
||||
Zone: "dc1",
|
||||
Capacity: *garage.NewNullableInt64(&capa),
|
||||
Tags: []string{ "fast", "amd64" },
|
||||
}},
|
||||
// Send a request
|
||||
resp, r, err := client.NodesApi.GetNodes(ctx).Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
|
||||
}
|
||||
staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute()
|
||||
msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute()
|
||||
fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured
|
||||
|
||||
health, _, _ := client.NodesApi.GetHealth(ctx).Execute()
|
||||
fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions)
|
||||
|
||||
// Key
|
||||
fmt.Println("\n--- key ---")
|
||||
key := "openapi-key"
|
||||
keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute()
|
||||
defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute()
|
||||
fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get())
|
||||
|
||||
id := *keyInfo.AccessKeyId
|
||||
canCreateBucket := true
|
||||
updateKeyRequest := *garage.NewUpdateKeyRequest()
|
||||
updateKeyRequest.SetName("openapi-key-updated")
|
||||
updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket })
|
||||
update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute()
|
||||
fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name)
|
||||
|
||||
keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute()
|
||||
fmt.Printf("Keys count: %v\n", len(keyList))
|
||||
|
||||
// Bucket
|
||||
fmt.Println("\n--- bucket ---")
|
||||
global_name := "global-ns-openapi-bucket"
|
||||
local_name := "local-ns-openapi-bucket"
|
||||
bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{
|
||||
GlobalAlias: &global_name,
|
||||
LocalAlias: &garage.CreateBucketRequestLocalAlias {
|
||||
AccessKeyId: keyInfo.AccessKeyId,
|
||||
Alias: &local_name,
|
||||
},
|
||||
}).Execute()
|
||||
defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute()
|
||||
fmt.Printf("Bucket id: %s\n", *bucketInfo.Id)
|
||||
|
||||
updateBucketRequest := *garage.NewUpdateBucketRequest()
|
||||
website := garage.NewUpdateBucketRequestWebsiteAccess()
|
||||
website.SetEnabled(true)
|
||||
website.SetIndexDocument("index.html")
|
||||
website.SetErrorDocument("errors/4xx.html")
|
||||
updateBucketRequest.SetWebsiteAccess(*website)
|
||||
quotas := garage.NewUpdateBucketRequestQuotas()
|
||||
quotas.SetMaxSize(1000000000)
|
||||
quotas.SetMaxObjects(999999999)
|
||||
updateBucketRequest.SetQuotas(*quotas)
|
||||
updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute()
|
||||
fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess)
|
||||
|
||||
bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute()
|
||||
fmt.Printf("Bucket count: %v\n", len(bucketList))
|
||||
// Process the response
|
||||
fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js.
|
|||
A short example:
|
||||
|
||||
```javascript
|
||||
const garage = require('garage_administration_api_v1garage_v0_9_0');
|
||||
const garage = require('garage_administration_api_v0garage_v0_8_0');
|
||||
|
||||
const api = new garage.ApiClient("http://127.0.0.1:3903/v1");
|
||||
const api = new garage.ApiClient("http://127.0.0.1:3903/v0");
|
||||
api.authentications['bearerAuth'].accessToken = "s3cr3t";
|
||||
|
||||
const [node, layout, key, bucket] = [
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ client = minio.Minio(
|
|||
"GKyourapikey",
|
||||
"abcd[...]1234",
|
||||
# Force the region, this is specific to garage
|
||||
region="garage",
|
||||
region="region",
|
||||
)
|
||||
```
|
||||
|
||||
|
|
@ -80,7 +80,7 @@ from garage_admin_sdk.apis import *
|
|||
from garage_admin_sdk.models import *
|
||||
|
||||
configuration = garage_admin_sdk.Configuration(
|
||||
host = "http://localhost:3903/v1",
|
||||
host = "http://localhost:3903/v0",
|
||||
access_token = "s3cr3t"
|
||||
)
|
||||
|
||||
|
|
@ -94,14 +94,13 @@ print(f"running garage {status.garage_version}, node_id {status.node}")
|
|||
|
||||
# Change layout of this node
|
||||
current = layout.get_layout()
|
||||
layout.add_layout([
|
||||
NodeRoleChange(
|
||||
id = status.node,
|
||||
layout.add_layout({
|
||||
status.node: NodeClusterInfo(
|
||||
zone = "dc1",
|
||||
capacity = 1000000000,
|
||||
capacity = 1,
|
||||
tags = [ "dev" ],
|
||||
)
|
||||
])
|
||||
})
|
||||
layout.apply_layout(LayoutVersion(
|
||||
version = current.version + 1
|
||||
))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Existing integrations"
|
||||
weight = 30
|
||||
weight = 3
|
||||
sort_by = "weight"
|
||||
template = "documentation.html"
|
||||
+++
|
||||
|
|
|
|||
|
|
@ -11,8 +11,7 @@ In this section, we cover the following web applications:
|
|||
| [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported |
|
||||
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
||||
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
||||
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
|
||||
| [Pixelfed](#pixelfed) | ✅ | Natively supported |
|
||||
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
||||
| [Pleroma](#pleroma) | ❓ | Not yet tested |
|
||||
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
|
||||
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
|
||||
|
|
@ -37,7 +36,7 @@ Second, we suppose you have created a key and a bucket.
|
|||
As a reminder, you can create a key for your nextcloud instance as follow:
|
||||
|
||||
```bash
|
||||
garage key create nextcloud-key
|
||||
garage key new --name nextcloud-key
|
||||
```
|
||||
|
||||
Keep the Key ID and the Secret key in a pad, they will be needed later.
|
||||
|
|
@ -69,7 +68,7 @@ $CONFIG = array(
|
|||
'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
|
||||
'port' => 3900, // Put your reverse proxy port or your S3 API port
|
||||
'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy
|
||||
'region' => 'garage', // Garage default region is named "garage", edit according to your cluster config
|
||||
'region' => 'garage', // Garage has only one region named "garage"
|
||||
'use_path_style' => true // Garage supports only path style, must be set to true
|
||||
],
|
||||
],
|
||||
|
|
@ -80,53 +79,6 @@ To test your new configuration, just reload your Nextcloud webpage and start sen
|
|||
|
||||
*External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html)
|
||||
|
||||
#### SSE-C encryption (since Garage v1.0)
|
||||
|
||||
Since version 1.0, Garage supports server-side encryption with customer keys
|
||||
(SSE-C). In this mode, Garage is responsible for encrypting and decrypting
|
||||
objects, but it does not store the encryption key itself. The encryption key
|
||||
should be provided by Nextcloud upon each request. This mode of operation is
|
||||
supported by Nextcloud and it has successfully been tested together with
|
||||
Garage.
|
||||
|
||||
To enable SSE-C encryption:
|
||||
|
||||
1. Make sure your Garage server is accessible via SSL through a reverse proxy
|
||||
such as Nginx, and that it is using a valid public certificate (Nextcloud
|
||||
might be able to connect to an S3 server that is using a self-signed
|
||||
certificate, but you will lose many hours while trying, so don't).
|
||||
Configure values for `use_ssl` and `port` accordingly in your `config.php`
|
||||
file.
|
||||
|
||||
2. Generate an encryption key using the following command:
|
||||
|
||||
```
|
||||
openssl rand -base64 32
|
||||
```
|
||||
|
||||
Make sure to keep this key **secret**!
|
||||
|
||||
3. Add the encryption key in your `config.php` file as follows:
|
||||
|
||||
|
||||
```php
|
||||
<?php
|
||||
$CONFIG = array(
|
||||
'objectstore' => [
|
||||
'class' => '\\OC\\Files\\ObjectStore\\S3',
|
||||
'arguments' => [
|
||||
...
|
||||
'sse_c_key' => 'exampleencryptionkeyLbU+5fKYQcVoqnn+RaIOXgo=',
|
||||
...
|
||||
],
|
||||
],
|
||||
```
|
||||
|
||||
Nextcloud will now make Garage encrypt files at rest in the storage bucket.
|
||||
These files will not be readable by an S3 client that has credentials to the
|
||||
bucket but doesn't also know the secret encryption key.
|
||||
|
||||
|
||||
### External Storage
|
||||
|
||||
**From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language).
|
||||
|
|
@ -135,7 +87,7 @@ bucket but doesn't also know the secret encryption key.
|
|||
*Click on the picture to zoom*
|
||||
|
||||
Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
|
||||
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
|
||||
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
|
||||
|
||||
Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
|
||||
|
||||
|
|
@ -186,15 +138,15 @@ a reasonable trade-off for some instances.
|
|||
Create a key for Peertube:
|
||||
|
||||
```bash
|
||||
garage key create peertube-key
|
||||
garage key new --name peertube-key
|
||||
```
|
||||
|
||||
Keep the Key ID and the Secret key in a pad, they will be needed later.
|
||||
|
||||
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
|
||||
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
|
||||
```bash
|
||||
garage bucket create peertube-videos
|
||||
garage bucket create peertube-playlists
|
||||
garage bucket create peertube-video
|
||||
garage bucket create peertube-playlist
|
||||
```
|
||||
|
||||
Now we allow our key to read and write on these buckets:
|
||||
|
|
@ -238,7 +190,7 @@ object_storage:
|
|||
# Put localhost only if you have a garage instance running on that node
|
||||
endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
|
||||
|
||||
# Garage default region is named "garage", edit according to your config
|
||||
# Garage supports only one region for now, named garage
|
||||
region: 'garage'
|
||||
|
||||
credentials:
|
||||
|
|
@ -253,7 +205,7 @@ object_storage:
|
|||
proxify_private_files: false
|
||||
|
||||
streaming_playlists:
|
||||
bucket_name: 'peertube-playlists'
|
||||
bucket_name: 'peertube-playlist'
|
||||
|
||||
# Keep it empty for our example
|
||||
prefix: ''
|
||||
|
|
@ -263,7 +215,7 @@ object_storage:
|
|||
|
||||
# Same settings but for webtorrent videos
|
||||
videos:
|
||||
bucket_name: 'peertube-videos'
|
||||
bucket_name: 'peertube-video'
|
||||
prefix: ''
|
||||
# You must fill this field to make Peertube use our reverse proxy/website logic
|
||||
base_url: 'http://peertube-videos.web.garage.localhost'
|
||||
|
|
@ -292,7 +244,7 @@ with average object size ranging from 50 KB to 150 KB.
|
|||
As such, your Garage cluster should be configured appropriately for good performance:
|
||||
|
||||
- use Garage v0.8.0 or higher with the [LMDB database engine](@documentation/reference-manual/configuration.md#db-engine-since-v0-8-0).
|
||||
Older versions of Garage used the Sled database engine which had issues, such as databases quickly ending up taking tens of GB of disk space.
|
||||
With the default Sled database engine, your database could quickly end up taking tens of GB of disk space.
|
||||
- the Garage database should be stored on a SSD
|
||||
|
||||
### Creating your bucket
|
||||
|
|
@ -300,7 +252,7 @@ As such, your Garage cluster should be configured appropriately for good perform
|
|||
This is the usual Garage setup:
|
||||
|
||||
```bash
|
||||
garage key create mastodon-key
|
||||
garage key new --name mastodon-key
|
||||
garage bucket create mastodon-data
|
||||
garage bucket allow mastodon-data --read --write --key mastodon-key
|
||||
```
|
||||
|
|
@ -335,7 +287,6 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t
|
|||
|
||||
```bash
|
||||
$ RAILS_ENV=production bin/tootctl media remove --days 3
|
||||
$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles
|
||||
$ RAILS_ENV=production bin/tootctl media remove-orphans
|
||||
$ RAILS_ENV=production bin/tootctl preview_cards remove --days 15
|
||||
```
|
||||
|
|
@ -354,6 +305,8 @@ Imports: 1.7 KB
|
|||
Settings: 0 Bytes
|
||||
```
|
||||
|
||||
Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567).
|
||||
|
||||
### Migrating your data
|
||||
|
||||
Data migration should be done with an efficient S3 client.
|
||||
|
|
@ -425,7 +378,7 @@ Supposing you have a working synapse installation, you can add the module with p
|
|||
Now create a bucket and a key for your matrix instance (note your Key ID and Secret Key somewhere, they will be needed later):
|
||||
|
||||
```bash
|
||||
garage key create matrix-key
|
||||
garage key new --name matrix-key
|
||||
garage bucket create matrix
|
||||
garage bucket allow matrix --read --write --key matrix-key
|
||||
```
|
||||
|
|
@ -441,7 +394,7 @@ media_storage_providers:
|
|||
store_synchronous: True # do we want to wait that the file has been written before returning?
|
||||
config:
|
||||
bucket: matrix # the name of our bucket, we chose matrix earlier
|
||||
region_name: garage # "garage" by default, edit according to your cluster config
|
||||
region_name: garage # only "garage" is supported for the region field
|
||||
endpoint_url: http://localhost:3900 # the path to the S3 endpoint
|
||||
access_key_id: "GKxxx" # your Key ID
|
||||
secret_access_key: "xxxx" # your Secret Key
|
||||
|
|
@ -467,7 +420,7 @@ Now we can write a simple script (eg `~/.local/bin/matrix-cache-gc`):
|
|||
## CONFIGURATION ##
|
||||
AWS_ACCESS_KEY_ID=GKxxx
|
||||
AWS_SECRET_ACCESS_KEY=xxxx
|
||||
AWS_ENDPOINT_URL=http://localhost:3900
|
||||
S3_ENDPOINT=http://localhost:3900
|
||||
S3_BUCKET=matrix
|
||||
MEDIA_STORE=/var/lib/matrix-synapse/media
|
||||
PG_USER=matrix
|
||||
|
|
@ -488,7 +441,7 @@ EOF
|
|||
|
||||
s3_media_upload update-db 1d
|
||||
s3_media_upload --no-progress check-deleted $MEDIA_STORE
|
||||
s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $AWS_ENDPOINT_URL
|
||||
s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $S3_ENDPOINT
|
||||
```
|
||||
|
||||
This script will list all the medias that were not accessed in the 24 hours according to your database.
|
||||
|
|
@ -521,52 +474,6 @@ And add a new line. For example, to run it every 10 minutes:
|
|||
|
||||
*External link:* [matrix-media-repo Documentation > S3](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html)
|
||||
|
||||
## ejabberd
|
||||
|
||||
ejabberd is an XMPP server implementation which, with the `mod_s3_upload`
|
||||
module in the [ejabberd-contrib](https://github.com/processone/ejabberd-contrib)
|
||||
repository, can be integrated to store chat media files in Garage.
|
||||
|
||||
For uploads, this module leverages presigned URLs - this allows XMPP clients to
|
||||
directly send media to Garage. Receiving clients then retrieve this media
|
||||
through the [static website](@/documentation/cookbook/exposing-websites.md)
|
||||
functionality.
|
||||
|
||||
As the data itself is publicly accessible to someone with knowledge of the
|
||||
object URL - users are recommended to use
|
||||
[E2EE](@/documentation/cookbook/encryption.md) to protect this data-at-rest
|
||||
from unauthorized access.
|
||||
|
||||
Install the module with:
|
||||
|
||||
```bash
|
||||
ejabberdctl module_install mod_s3_upload
|
||||
```
|
||||
|
||||
Create the required key and bucket with:
|
||||
|
||||
```bash
|
||||
garage key new --name ejabberd
|
||||
garage bucket create objects.xmpp-server.fr
|
||||
garage bucket allow objects.xmpp-server.fr --read --write --key ejabberd
|
||||
garage bucket website --allow objects.xmpp-server.fr
|
||||
```
|
||||
|
||||
The module can then be configured with:
|
||||
|
||||
```
|
||||
mod_s3_upload:
|
||||
#bucket_url: https://objects.xmpp-server.fr.my-garage-instance.mydomain.tld
|
||||
bucket_url: https://my-garage-instance.mydomain.tld/objects.xmpp-server.fr
|
||||
access_key_id: GK...
|
||||
access_key_secret: ...
|
||||
region: garage
|
||||
download_url: https://objects.xmpp-server.fr
|
||||
```
|
||||
|
||||
Other configuration options can be found in the
|
||||
[configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
|
||||
|
||||
## Pixelfed
|
||||
|
||||
[Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
|
||||
|
|
@ -632,7 +539,7 @@ secret_key = 'abcdef0123456789...'
|
|||
|
||||
```
|
||||
PICTRS__STORE__TYPE=object_storage
|
||||
PICTRS__STORE__ENDPOINT=http://my-garage-instance.mydomain.tld:3900
|
||||
PICTRS__STORE__ENDPOINT=http:/my-garage-instance.mydomain.tld:3900
|
||||
PICTRS__STORE__BUCKET_NAME=pictrs-data
|
||||
PICTRS__STORE__REGION=garage
|
||||
PICTRS__STORE__ACCESS_KEY=GK...
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue