mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-10 08:09:17 -05:00
Compare commits
528 changed files with 4429 additions and 103995 deletions
|
|
@ -1,17 +0,0 @@
|
|||
# Version and commit come in as build args, so the build doesn't need .git.
|
||||
.git
|
||||
.github
|
||||
|
||||
# Local build output and runtime data
|
||||
/proxy
|
||||
dist/
|
||||
build/
|
||||
cache/
|
||||
*.db
|
||||
*.tar.gz
|
||||
|
||||
# Local config and secrets
|
||||
.env
|
||||
.env.local
|
||||
proxy.cfg
|
||||
config.yaml
|
||||
49
.github/workflows/ci.yml
vendored
49
.github/workflows/ci.yml
vendored
|
|
@ -31,40 +31,6 @@ jobs:
|
|||
- name: Test
|
||||
run: go test -v -race ./...
|
||||
|
||||
postgres:
|
||||
name: test (postgres)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:b8faeff7712663c2773a190181e1246e37e364b887611c04658c28b44e2408fe
|
||||
env:
|
||||
POSTGRES_PASSWORD: test
|
||||
POSTGRES_DB: proxy
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d proxy"
|
||||
--health-interval 2s
|
||||
--health-timeout 5s
|
||||
--health-retries 30
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
# The packages with tests that need PROXY_DATABASE_URL. -p 1 because
|
||||
# their tests drop and recreate tables in the one shared database.
|
||||
- name: Test
|
||||
env:
|
||||
PROXY_DATABASE_URL: postgres://postgres:test@localhost:5432/proxy?sslmode=disable
|
||||
run: go test -v -race -p 1 ./internal/database/... ./internal/handler/... ./internal/server/...
|
||||
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
|
@ -77,11 +43,8 @@ jobs:
|
|||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0
|
||||
|
||||
- name: golangci-lint
|
||||
run: golangci-lint run ./...
|
||||
run: go tool golangci-lint run ./...
|
||||
|
||||
helm:
|
||||
name: Helm chart
|
||||
|
|
@ -110,13 +73,3 @@ jobs:
|
|||
--set 'ingress.hosts[0].paths[0].path=/' \
|
||||
--set 'ingress.hosts[0].paths[0].pathType=Prefix' \
|
||||
>/dev/null
|
||||
helm template proxy deploy/charts/proxy \
|
||||
--set 'extraVolumes[0].name=corp-ca' \
|
||||
--set 'extraVolumes[0].configMap.name=corp-ca' \
|
||||
--set 'extraVolumeMounts[0].name=corp-ca' \
|
||||
--set 'extraVolumeMounts[0].mountPath=/etc/proxy/ca' \
|
||||
--set 'extraVolumeMounts[0].readOnly=true' \
|
||||
--set 'extraInitContainers[0].name=wait-for-db' \
|
||||
--set 'extraInitContainers[0].image=busybox:1.37' \
|
||||
--set 'extraInitContainers[0].command={sh,-c,exit 0}' \
|
||||
>/dev/null
|
||||
|
|
|
|||
137
.github/workflows/monthly-release.yml
vendored
137
.github/workflows/monthly-release.yml
vendored
|
|
@ -1,137 +0,0 @@
|
|||
name: Monthly release
|
||||
|
||||
# Cuts a release on the 1st of each month when main is green and something
|
||||
# has merged since the last release: a patch release, or a new minor version
|
||||
# if a pull request labelled "breaking" was merged (a new major version from
|
||||
# 1.0 on). A breaking release is created as a draft, and its images and chart
|
||||
# are published only when someone publishes the draft, after writing the
|
||||
# upgrade notes. Scheduled runs only report what they would do until the
|
||||
# repository variable MONTHLY_RELEASE is "true".
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 1 * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Only report what would be released
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: monthly-release
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
checks: read
|
||||
pull-requests: read
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Decide whether to release
|
||||
id: plan
|
||||
env:
|
||||
EVENT: ${{ github.event_name }}
|
||||
INPUT_DRY_RUN: ${{ inputs.dry_run }}
|
||||
ENABLED: ${{ vars.MONTHLY_RELEASE }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$EVENT" = workflow_dispatch ]; then
|
||||
dry_run=$INPUT_DRY_RUN
|
||||
elif [ "$ENABLED" = true ]; then
|
||||
dry_run=false
|
||||
else
|
||||
dry_run=true
|
||||
fi
|
||||
echo "dry_run=$dry_run" >> "$GITHUB_OUTPUT"
|
||||
|
||||
sha=$(git rev-parse HEAD)
|
||||
last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*')
|
||||
# A breaking release waits as a draft for its upgrade notes. Building
|
||||
# on its tag would publish those changes without them. Older tags
|
||||
# were released by hand and may have no release at all.
|
||||
draft=$(gh release view "$last" -R "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo false)
|
||||
if [ "$draft" = true ]; then
|
||||
echo "::warning::$last is still a draft, publish it first"
|
||||
exit 0
|
||||
fi
|
||||
merged=$(git rev-list --count "$last..$sha")
|
||||
if [ "$merged" -eq 0 ]; then
|
||||
echo "Nothing merged since $last"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Every check on main's HEAD must have finished green. This run
|
||||
# shows up as a check on the same commit, so it is left out.
|
||||
bad=$(gh api --paginate "repos/$REPO/commits/$sha/check-runs?per_page=100" \
|
||||
--jq ".check_runs[]
|
||||
| select(.html_url | contains(\"/actions/runs/$GITHUB_RUN_ID/\") | not)
|
||||
| select(.status != \"completed\"
|
||||
or (.conclusion | IN(\"success\", \"skipped\", \"neutral\") | not))
|
||||
| .name")
|
||||
if [ -n "$bad" ]; then
|
||||
echo "::warning::main ($sha) is not green, skipping: $(echo "$bad" | sort -u | paste -sd, -)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
since=$(git log -1 --format=%cI "$last")
|
||||
breaking=$(gh pr list -R "$REPO" --state merged --base main --limit 1000 \
|
||||
--search "merged:>$since label:breaking" --json number --jq '.[].number')
|
||||
|
||||
IFS=. read -r major minor patch <<<"${last#v}"
|
||||
if [ -z "$breaking" ]; then
|
||||
tag="v$major.$minor.$((patch + 1))"
|
||||
else
|
||||
echo "Breaking changes merged since $last: #$(echo "$breaking" | sort -n | paste -sd, - | sed 's/,/, #/g')"
|
||||
if [ "$major" -eq 0 ]; then
|
||||
tag="v0.$((minor + 1)).0"
|
||||
else
|
||||
tag="v$((major + 1)).0.0"
|
||||
fi
|
||||
fi
|
||||
echo "$merged commits since $last, next release $tag at $sha"
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "draft=$([ -n "$breaking" ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the
|
||||
# release workflows are dispatched on the new tag instead.
|
||||
- name: Tag and start the release workflows
|
||||
if: steps.plan.outputs.tag != ''
|
||||
env:
|
||||
TAG: ${{ steps.plan.outputs.tag }}
|
||||
SHA: ${{ steps.plan.outputs.sha }}
|
||||
DRY_RUN: ${{ steps.plan.outputs.dry_run }}
|
||||
DRAFT: ${{ steps.plan.outputs.draft }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$DRAFT" = true ]; then
|
||||
what="start release.yml for a draft release"
|
||||
else
|
||||
what="start release.yml and publish.yml"
|
||||
fi
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "Dry run: would tag $TAG at $SHA and $what"
|
||||
exit 0
|
||||
fi
|
||||
gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null
|
||||
gh workflow run release.yml -R "$REPO" --ref "$TAG" -f draft="$DRAFT"
|
||||
if [ "$DRAFT" = true ]; then
|
||||
echo "::notice::$TAG is a draft release: add the upgrade notes and publish it to push the images and chart"
|
||||
else
|
||||
gh workflow run publish.yml -R "$REPO" --ref "$TAG"
|
||||
fi
|
||||
echo "Tagged $TAG at $SHA"
|
||||
9
.github/workflows/publish.yml
vendored
9
.github/workflows/publish.yml
vendored
|
|
@ -5,10 +5,6 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
# A draft release made by monthly-release.yml is published by hand. Releases
|
||||
# published with GITHUB_TOKEN don't start this.
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions: {}
|
||||
|
||||
|
|
@ -38,7 +34,7 @@ jobs:
|
|||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
|
@ -61,9 +57,6 @@ jobs:
|
|||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ github.sha }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
|
||||
|
|
|
|||
12
.github/workflows/release.yml
vendored
12
.github/workflows/release.yml
vendored
|
|
@ -4,13 +4,6 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
# Started by monthly-release.yml on the tag it creates.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
draft:
|
||||
description: Create the release as a draft
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
|
@ -18,7 +11,6 @@ permissions:
|
|||
|
||||
jobs:
|
||||
release:
|
||||
if: github.ref_type == 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
|
@ -35,9 +27,9 @@ jobs:
|
|||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0
|
||||
- uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
args: release --clean${{ inputs.draft && ' --draft' || '' }}
|
||||
args: release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
|
|||
2
.github/workflows/swagger.yml
vendored
2
.github/workflows/swagger.yml
vendored
|
|
@ -22,7 +22,7 @@ jobs:
|
|||
go-version-file: go.mod
|
||||
|
||||
- name: Install swag
|
||||
run: go install github.com/swaggo/swag/cmd/swag@v1.16.6
|
||||
run: go install github.com/swaggo/swag/cmd/swag@latest
|
||||
|
||||
- name: Generate swagger
|
||||
run: go generate ./internal/server
|
||||
|
|
|
|||
2
.github/workflows/zizmor.yml
vendored
2
.github/workflows/zizmor.yml
vendored
|
|
@ -34,4 +34,4 @@ jobs:
|
|||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0
|
||||
uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
|
||||
|
|
|
|||
|
|
@ -26,7 +26,3 @@ linters:
|
|||
- goconst
|
||||
- dupl
|
||||
- mnd
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ builds:
|
|||
- linux
|
||||
- darwin
|
||||
- windows
|
||||
- freebsd
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
|
|
|
|||
|
|
@ -160,7 +160,6 @@ mux.Handle("/newregistry/", http.StripPrefix("/newregistry", newHandler.Routes()
|
|||
- Keep functions short and focused
|
||||
- Write tests for new functionality
|
||||
- Document exported types and functions
|
||||
- A new Prometheus metric needs a tile on `/ui/analytics` and an entry in `metricSurface` (`internal/server/analytics_coverage_test.go`). The page is meant to be a complete view of `/metrics`, so `TestEveryMetricIsSurfaced` fails until both exist.
|
||||
|
||||
## Testing
|
||||
|
||||
|
|
@ -212,24 +211,6 @@ Add PyPI registry support
|
|||
- Add tests for wheel and sdist downloads
|
||||
```
|
||||
|
||||
## Reviewing and Merging
|
||||
|
||||
Maintainers are the members of the proxy team in the git-pkgs org. Any maintainer can review and merge. A pull request needs green CI and an approval from a maintainer other than its author. A "request changes" review from any maintainer blocks it until that reviewer approves or withdraws the review.
|
||||
|
||||
Using AI to write or review is fine, but whoever approves a pull request should understand it and stand behind it. If it touches an ecosystem you don't use, you can ask someone who does to look before you approve.
|
||||
|
||||
Open an issue before a bigger change: anything that changes behaviour users rely on, new configuration that would be hard to remove, or a change that fans out into other git-pkgs modules. If no maintainer objects within a few days, open the pull request; it's reviewed like any other. Objections are worked out in the issue, with @andrew breaking ties for now. Where possible, deprecate before removing: an old configuration key keeps working, with a warning, in at least one release before the one that removes it.
|
||||
|
||||
## Releases
|
||||
|
||||
A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on.
|
||||
|
||||
A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. Until then the monthly run releases nothing.
|
||||
|
||||
Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default.
|
||||
|
||||
To release by hand, push a `v*` tag.
|
||||
|
||||
## Questions?
|
||||
|
||||
Open an issue on GitHub if you have questions or need help getting started.
|
||||
|
|
|
|||
16
Dockerfile
16
Dockerfile
|
|
@ -1,4 +1,4 @@
|
|||
FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS builder
|
||||
FROM --platform=$BUILDPLATFORM golang:1.26.7-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
|
|
@ -14,15 +14,11 @@ COPY . .
|
|||
|
||||
# Build the binary for the target platform
|
||||
ARG TARGETARCH
|
||||
ARG VERSION=dev
|
||||
ARG COMMIT=unknown
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
|
||||
-ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \
|
||||
-o /proxy ./cmd/proxy
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||
|
||||
FROM alpine:3.24.3
|
||||
FROM alpine:3.24.1
|
||||
|
||||
RUN apk add --no-cache ca-certificates git
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY --from=builder /proxy /usr/local/bin/proxy
|
||||
|
||||
|
|
@ -35,9 +31,5 @@ WORKDIR /data
|
|||
|
||||
EXPOSE 8080
|
||||
|
||||
# /health checks the database and storage and can take up to ten seconds.
|
||||
HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1
|
||||
|
||||
ENTRYPOINT ["proxy"]
|
||||
CMD ["serve"]
|
||||
|
|
|
|||
644
README.md
644
README.md
|
|
@ -20,8 +20,6 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build
|
|||
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference.
|
||||
|
||||
One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy.
|
||||
|
||||
## Artifact Scanning
|
||||
|
||||
Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients.
|
||||
|
|
@ -49,11 +47,11 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi
|
|||
| npm | JavaScript | Yes | ✓ |
|
||||
| Cargo | Rust | Yes | ✓ |
|
||||
| RubyGems | Ruby | Yes | ✓ |
|
||||
| Go proxy | Go | Yes† | ✓ |
|
||||
| Go proxy | Go | | ✓ |
|
||||
| Hex | Elixir | Yes* | ✓ |
|
||||
| pub.dev | Dart | Yes | ✓ |
|
||||
| PyPI | Python | Yes | ✓ |
|
||||
| Maven | Java | Yes‡ | ✓ |
|
||||
| Maven | Java | | ✓ |
|
||||
| Gradle Build Cache | Java/Kotlin | | ✓ |
|
||||
| NuGet | .NET | Yes | ✓ |
|
||||
| Composer | PHP | Yes | ✓ |
|
||||
|
|
@ -67,21 +65,15 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi
|
|||
| Debian | Debian/Ubuntu | | ✓ |
|
||||
| RPM | RHEL/Fedora | | ✓ |
|
||||
| Alpine | Alpine Linux | | ✓ |
|
||||
| Arch | Arch Linux | | ✓ |
|
||||
| FreeBSD | FreeBSD (ports and pkgbase) | | ✓ |
|
||||
| Chef Supermarket | Chef | | ✓ |
|
||||
| Arch | Arch Linux | | ✗ |
|
||||
| Chef | Chef | | ✗ |
|
||||
| Generic | Any | | ✓ |
|
||||
| Git | Any | | ✓ |
|
||||
| Helm | Kubernetes | Yes | ✓ |
|
||||
| Helm | Kubernetes | | ✓ |
|
||||
| Vagrant | Vagrant | | ✗ |
|
||||
|
||||
Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet.
|
||||
|
||||
\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled.
|
||||
|
||||
† Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed.
|
||||
|
||||
‡ Maven cooldown uses the `Last-Modified` header of each version's `.pom`. On Maven Central that is the deploy time; a Nexus or Artifactory upstream may report when it cached the file instead. SNAPSHOT versions are not held back.
|
||||
\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload.
|
||||
|
||||
## Install
|
||||
|
||||
|
|
@ -89,11 +81,7 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in
|
|||
brew install git-pkgs/git-pkgs/proxy
|
||||
```
|
||||
|
||||
Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image:
|
||||
|
||||
```bash
|
||||
docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest
|
||||
```
|
||||
Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases).
|
||||
|
||||
### Helm
|
||||
|
||||
|
|
@ -110,33 +98,6 @@ using SQLite and filesystem artifact storage under `/data`. See
|
|||
[`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress,
|
||||
external database and object-storage configuration options.
|
||||
|
||||
The SQLite database needs a local disk or a block-backed volume, even with one
|
||||
replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class
|
||||
can still be NFS. If the cluster only offers network filesystem storage, use
|
||||
Postgres. Artifact storage can stay on NFS, separately from the database. See
|
||||
[SQLite](docs/configuration.md#sqlite).
|
||||
|
||||
To trust a private CA, for example when upstreams sit behind a TLS-intercepting
|
||||
corporate proxy or are internal registries signed by an in-house CA, mount the
|
||||
certificate with `extraVolumes` and `extraVolumeMounts` and add its directory
|
||||
to `SSL_CERT_DIR`. The proxy process makes all upstream TLS connections itself,
|
||||
Git fetches included, so no other configuration is needed. Public CAs stay
|
||||
trusted alongside the private one:
|
||||
|
||||
```yaml
|
||||
extraVolumes:
|
||||
- name: corp-ca
|
||||
configMap:
|
||||
name: corp-ca # e.g. key corp-ca.crt, PEM-encoded
|
||||
extraVolumeMounts:
|
||||
- name: corp-ca
|
||||
mountPath: /etc/proxy/ca
|
||||
readOnly: true
|
||||
extraEnv:
|
||||
- name: SSL_CERT_DIR
|
||||
value: /etc/ssl/certs:/etc/proxy/ca
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
|
|
@ -150,8 +111,7 @@ go build -o proxy ./cmd/proxy
|
|||
./proxy -listen :3000 -base-url https://proxy.example.com
|
||||
```
|
||||
|
||||
The proxy is now running. Configure your package managers to use it. For image
|
||||
builds, see [Using proxy in Docker builds](docs/docker-builds.md).
|
||||
The proxy is now running. Configure your package managers to use it.
|
||||
|
||||
## OpenAPI (Swagger)
|
||||
|
||||
|
|
@ -199,12 +159,7 @@ through the proxy: the audit and signing-key endpoints are passed through to the
|
|||
configured upstream registry, with upstream authentication applied. Advisories
|
||||
therefore come from upstream's database, not from the proxy's own vulnerability
|
||||
data, and versions withheld by [cooldown](#version-cooldown) are not excluded
|
||||
from the report. Packuments point each version's attestations (provenance) at
|
||||
the proxy, which fetches them from the package's registry and caches them;
|
||||
attestations of versions held back by cooldown or blocked by the denylist are
|
||||
not served.
|
||||
|
||||
To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds).
|
||||
from the report.
|
||||
|
||||
### Cargo
|
||||
|
||||
|
|
@ -245,24 +200,6 @@ export GOPROXY=http://localhost:8080/go,direct
|
|||
|
||||
Or in your shell profile for persistence.
|
||||
|
||||
Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at
|
||||
`/go/sumdb/sum.golang.org/`, and the `go` command uses it without further
|
||||
setup, so clients need no route of their own to `sum.golang.org`. If the
|
||||
proxy loses `https://sum.golang.org`, it keeps answering until its cached
|
||||
copy of the database's signed tree is older than `metadata_ttl`; after that
|
||||
the `go` command contacts the database directly as before. Set
|
||||
`upstream.go_sumdbs_offline` on networks where clients cannot do that. Other
|
||||
checksum databases are listed under `upstream.go_sumdbs`, see
|
||||
[Go checksum database](docs/configuration.md#go-checksum-database).
|
||||
|
||||
govulncheck can use the proxy for the Go vulnerability database:
|
||||
|
||||
```bash
|
||||
govulncheck -db http://localhost:8080/go/vulndb ./...
|
||||
```
|
||||
|
||||
See [Go vulnerability database](docs/configuration.md#go-vulnerability-database).
|
||||
|
||||
### Homebrew
|
||||
|
||||
Point Homebrew's JSON API and artifact domain at the proxy:
|
||||
|
|
@ -274,7 +211,7 @@ export HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080
|
|||
|
||||
The artifact domain proxies manifests and bottle blobs under `/v2/homebrew/core/`. GHCR routing is limited to that repository. Source archives, cask application downloads, custom tap artifacts, and legacy flat-file bottle mirrors use Homebrew's normal fallback URLs. Keep fallback enabled by leaving `HOMEBREW_ARTIFACT_DOMAIN_NO_FALLBACK` unset.
|
||||
|
||||
Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting. `proxy mirror pkg:brew/<formula>` pre-fetches a formula's bottles (see [Homebrew bottles](docs/configuration.md#homebrew-bottles)).
|
||||
Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting.
|
||||
|
||||
The upstreams default to `https://formulae.brew.sh/api` for the JSON API and `https://ghcr.io` for artifacts. To chain this proxy to another proxy, configure its Homebrew endpoints as the upstreams:
|
||||
|
||||
|
|
@ -288,10 +225,10 @@ The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROX
|
|||
|
||||
### Hex (Elixir)
|
||||
|
||||
Configure the mirror:
|
||||
Configure in `~/.hex/hex.config`:
|
||||
|
||||
```bash
|
||||
mix hex.config mirror_url http://localhost:8080/hex
|
||||
```erlang
|
||||
{default_url, <<"http://localhost:8080/hex">>}.
|
||||
```
|
||||
|
||||
Or set the environment variable:
|
||||
|
|
@ -300,14 +237,6 @@ Or set the environment variable:
|
|||
export HEX_MIRROR=http://localhost:8080/hex
|
||||
```
|
||||
|
||||
When the proxy applies a cooldown, also set:
|
||||
|
||||
```bash
|
||||
export HEX_UNSAFE_REGISTRY=1
|
||||
```
|
||||
|
||||
Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too.
|
||||
|
||||
### pub.dev (Dart/Flutter)
|
||||
|
||||
Set the PUB_HOSTED_URL environment variable:
|
||||
|
|
@ -349,8 +278,6 @@ Add to your `~/.m2/settings.xml`:
|
|||
|
||||
The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found.
|
||||
|
||||
Further Maven repositories can be configured under `upstream.maven_repositories` and are served at `/maven/{name}/`, each from its own upstream (see [Maven repositories](docs/configuration.md#maven-repositories)). Point a second `<repository>` or Gradle `maven(url = ...)` entry at `http://localhost:8080/maven/{name}/`.
|
||||
|
||||
For Gradle plugin resolution via the same proxy endpoint:
|
||||
|
||||
```kotlin
|
||||
|
|
@ -363,20 +290,20 @@ pluginManagement {
|
|||
|
||||
### Gradle HTTP Build Cache
|
||||
|
||||
Configure in `settings.gradle.kts`:
|
||||
Configure in `settings.gradle(.kts)`:
|
||||
|
||||
```kotlin
|
||||
buildCache {
|
||||
local {
|
||||
enabled = false
|
||||
}
|
||||
remote<HttpBuildCache> {
|
||||
url = uri("http://localhost:8080/gradle/")
|
||||
push = true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default.
|
||||
|
||||
To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote<HttpBuildCache>` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache).
|
||||
|
||||
### NuGet
|
||||
|
||||
Configure in `nuget.config`:
|
||||
|
|
@ -417,17 +344,6 @@ Or set globally:
|
|||
composer config -g repositories.proxy composer http://localhost:8080/composer
|
||||
```
|
||||
|
||||
`composer audit`, `audit.block-insecure` and Packagist's malware filter list
|
||||
work through the proxy when `upstream.composer_repository` offers them in its
|
||||
`packages.json`, as repo.packagist.org does. The proxy advertises them in its
|
||||
own `packages.json` and passes the advisories and filter list requests through
|
||||
to the upstream uncached. Names of packages in `upstream.composer_routes` or
|
||||
`hosted.composer_vendors` are removed from advisories requests before they are
|
||||
sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from
|
||||
the report.
|
||||
|
||||
To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds).
|
||||
|
||||
### Conan (C/C++)
|
||||
|
||||
Add the proxy as a remote:
|
||||
|
|
@ -439,24 +355,6 @@ conan remote disable conancenter
|
|||
|
||||
Or configure in `~/.conan2/remotes.json`.
|
||||
|
||||
### Chef Supermarket (Berkshelf, Policyfiles)
|
||||
|
||||
Point Berkshelf at the proxy in your `Berksfile`:
|
||||
|
||||
```ruby
|
||||
source "http://localhost:8080/chef"
|
||||
```
|
||||
|
||||
Or set the default source of a Policyfile:
|
||||
|
||||
```ruby
|
||||
default_source :supermarket, "http://localhost:8080/chef"
|
||||
```
|
||||
|
||||
The proxy serves the `/universe` dependency index and the cookbook API, rewriting every download location to itself while leaving dependency constraints unchanged. Cookbook archives are fetched through the Supermarket download redirect and cached, so they stay available when the Supermarket is down. With `cache_metadata` on, the universe and cookbook metadata follow the usual TTL and stale fallback.
|
||||
|
||||
To proxy a private Supermarket instead of `supermarket.chef.io`, set `upstream.chef` (or `PROXY_UPSTREAM_CHEF`) and add credentials under `upstream.auth` if it needs them. See [docs/configuration.md](docs/configuration.md#upstream-registries).
|
||||
|
||||
### Conda
|
||||
|
||||
Configure in `~/.condarc`:
|
||||
|
|
@ -469,17 +367,12 @@ default_channels:
|
|||
- http://localhost:8080/conda/main
|
||||
```
|
||||
|
||||
`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it.
|
||||
|
||||
Or set via command:
|
||||
|
||||
```bash
|
||||
conda config --add channels http://localhost:8080/conda/main
|
||||
conda config --add default_channels http://localhost:8080/conda/main
|
||||
```
|
||||
|
||||
pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on.
|
||||
|
||||
### CRAN (R)
|
||||
|
||||
Set the repository in R:
|
||||
|
|
@ -498,8 +391,6 @@ local({
|
|||
})
|
||||
```
|
||||
|
||||
Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata.
|
||||
|
||||
### Julia
|
||||
|
||||
Set the Pkg server before starting Julia:
|
||||
|
|
@ -556,79 +447,6 @@ Or pull images directly:
|
|||
docker pull localhost:8080/library/nginx:latest
|
||||
```
|
||||
|
||||
The registry is pull-only. BuildKit cache exports (`--cache-to type=registry`)
|
||||
and other pushes need a separate registry.
|
||||
|
||||
SBOMs, signatures and attestations attached to an image are found through the
|
||||
OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover`
|
||||
and `notation verify` use it, and so does cosign for its newer signature
|
||||
bundles. The proxy forwards these requests and caches the returned index with
|
||||
the same TTL as other metadata, so they keep working offline. It does not
|
||||
filter by `artifactType`; it returns the full list and the client filters it,
|
||||
as the spec allows. Registries without the referrers API (GHCR, for example)
|
||||
answer 404, and clients then look up the `sha256-<digest>` tag instead, which
|
||||
the proxy serves like any other manifest. The proxy is pull-only, so attaching
|
||||
new artifacts through it is not supported.
|
||||
|
||||
#### containerd (Kubernetes, k3s, nerdctl)
|
||||
|
||||
containerd mirrors send the original registry host in an `ns` query
|
||||
parameter, so one mirror entry can serve Docker Hub and the registries
|
||||
configured in `upstream.oci` whose URL has no path. Point containerd's CRI
|
||||
plugin at a hosts directory in `/etc/containerd/config.toml` and restart
|
||||
containerd. For containerd 2.x:
|
||||
|
||||
```toml
|
||||
version = 3
|
||||
|
||||
[plugins."io.containerd.cri.v1.images".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
```
|
||||
|
||||
For containerd 1.x:
|
||||
|
||||
```toml
|
||||
version = 2
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
```
|
||||
|
||||
Then create `/etc/containerd/certs.d/_default/hosts.toml`:
|
||||
|
||||
```toml
|
||||
[host."http://proxy.example.com:8080"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
```
|
||||
|
||||
To check that CRI picked up the directory, confirm the setting containerd
|
||||
runs with and pull through CRI (on k3s use `k3s crictl`):
|
||||
|
||||
```bash
|
||||
containerd config dump | grep config_path
|
||||
crictl pull docker.io/library/nginx:latest
|
||||
```
|
||||
|
||||
The proxy logs a `container manifest request` line for the pull and, when
|
||||
`access_log.path` is set, an access-log entry. `ctr images pull --hosts-dir`
|
||||
is no substitute for this check: it reads the directory itself and succeeds
|
||||
even while CRI still has no `config_path`.
|
||||
|
||||
`docker.io` and the host of `upstream.oci_default` use the default registry;
|
||||
the host of each `upstream.oci` URL uses that named registry, and so do the
|
||||
hosts listed for it in `upstream.oci_mirrors`. Pulls for any other registry get
|
||||
`404 NAME_UNKNOWN`, and containerd falls back to the registry itself. Existing
|
||||
per-registry `hosts.toml` files that point at `/v2/upstream/{name}` with
|
||||
`override_path = true` keep working for that upstream's own host and for
|
||||
Docker Hub. If the upstream is a mirror of the registry the nodes pull from,
|
||||
say an Artifactory remote of `ghcr.io`, list that registry in
|
||||
`upstream.oci_mirrors` (`ghcr: ["ghcr.io"]`); otherwise those pulls get the
|
||||
404 as well. See [docs/configuration.md](docs/configuration.md) for how
|
||||
hosts are matched and which entry wins when two share a host. k3s generates
|
||||
the hosts directory itself from `/etc/rancher/k3s/registries.yaml`; `mirrors:
|
||||
{"*": {endpoint: ["http://proxy.example.com:8080"]}}` produces the same
|
||||
`_default` entry.
|
||||
|
||||
### Helm
|
||||
|
||||
Configure each HTTP chart repository with a name, then add the matching proxy
|
||||
|
|
@ -670,31 +488,12 @@ Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`:
|
|||
deb http://localhost:8080/debian stable main contrib
|
||||
```
|
||||
|
||||
Or in deb822 format, in `/etc/apt/sources.list.d/proxy.sources`:
|
||||
|
||||
```
|
||||
Types: deb
|
||||
URIs: http://localhost:8080/debian
|
||||
Suites: stable
|
||||
Components: main contrib
|
||||
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
|
||||
```
|
||||
|
||||
Replace your existing sources.list entries (on newer releases also
|
||||
`debian.sources` or `ubuntu.sources` in `/etc/apt/sources.list.d/`), then:
|
||||
Replace your existing sources.list entries, then:
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
```
|
||||
|
||||
There is no proxy key to install. The proxy serves the upstream's signed
|
||||
`InRelease` and `Release` files unchanged, so APT verifies them with the
|
||||
upstream archive's own key from the distribution's keyring package:
|
||||
`debian-archive-keyring` on Debian, `ubuntu-keyring` on Ubuntu
|
||||
(`/usr/share/keyrings/ubuntu-archive-keyring.gpg`). `Signed-By:` in a
|
||||
`.sources` file, or `[signed-by=...]` in a `.list` line, restricts the source
|
||||
to that key. For a third-party upstream, use the key its vendor publishes.
|
||||
|
||||
The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment:
|
||||
|
||||
```yaml
|
||||
|
|
@ -727,20 +526,16 @@ indexes unchanged. To keep APT from selecting a denied version, configure
|
|||
|
||||
### RPM / Yum / DNF
|
||||
|
||||
In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with:
|
||||
Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`:
|
||||
|
||||
```ini
|
||||
[proxy-fedora]
|
||||
name=Fedora via Proxy
|
||||
baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
```
|
||||
|
||||
In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with:
|
||||
|
||||
```ini
|
||||
baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/
|
||||
```
|
||||
|
||||
Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid.
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
|
|
@ -770,11 +565,6 @@ signature verification keeps working. Indexes use the metadata cache
|
|||
(`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared
|
||||
artifact cache and remain available when the upstream is unreachable.
|
||||
|
||||
Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve
|
||||
them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions
|
||||
stay installable after the mirror moves on. See
|
||||
[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots).
|
||||
|
||||
To proxy other mirrors or private repositories, configure named upstreams
|
||||
under `upstream.apk` (this replaces the built-in default; re-add `alpine` if
|
||||
you still want it):
|
||||
|
|
@ -793,127 +583,6 @@ http://localhost:8080/apk/private
|
|||
apk appends the architecture and index filename to each repository line
|
||||
itself.
|
||||
|
||||
### Arch Linux / pacman
|
||||
|
||||
Put the proxy at the top of `/etc/pacman.d/mirrorlist`. The default repository
|
||||
name `archlinux` proxies the official GeoIP mirror
|
||||
(`https://geo.mirror.pkgbuild.com`):
|
||||
|
||||
```
|
||||
Server = http://localhost:8080/arch/archlinux/$repo/os/$arch
|
||||
```
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
pacman -Syu
|
||||
```
|
||||
|
||||
Repository databases (`core.db`, `core.files` and their `.tar.*` forms),
|
||||
database signatures, packages and package signatures are served byte-for-byte
|
||||
unchanged, so pacman's signature checks keep working. Databases use the
|
||||
metadata cache (`metadata_ttl`, stale fallback). Each database refresh also
|
||||
fetches its signature and stores it for exactly that database, so a signature
|
||||
is only ever served with the database it signs. Enable `cache_metadata` (or set `PROXY_CACHE_METADATA=true`) so
|
||||
`pacman -Sy` keeps working while the upstream is down; without it databases are
|
||||
fetched fresh on every request. Packages and their `.sig` files are stored in the shared artifact
|
||||
cache and remain available when the upstream is unreachable.
|
||||
|
||||
The proxy only starts sending a package once it has the whole file, and pacman
|
||||
gives up on a download that sends nothing for 10 seconds. The fetch keeps
|
||||
running and the next try is served from the cache. On a slow upstream, set
|
||||
`DisableDownloadTimeout` in `/etc/pacman.conf` to avoid the failed first try.
|
||||
|
||||
`extra.files` is about 50 MB and is read into memory on each `pacman -F`
|
||||
request. Raise `metadata_max_size` (100 MB by default) for larger custom
|
||||
repositories.
|
||||
|
||||
A custom repository that rebuilds a package under the same version leaves the
|
||||
old package and signature in the cache. Bump `pkgrel` instead, or remove the
|
||||
cached version.
|
||||
|
||||
#### Arch Linux ARM
|
||||
|
||||
Arch Linux ARM has no built-in repository. Add one under `upstream.arch`, using a
|
||||
regional mirror from <https://archlinuxarm.org/about/mirrors> that serves HTTPS:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
arch:
|
||||
archlinux: "https://geo.mirror.pkgbuild.com"
|
||||
archlinuxarm: "https://<region>.mirror.archlinuxarm.org"
|
||||
```
|
||||
|
||||
Setting `upstream.arch` replaces the built-in default, so keep `archlinux` in the map
|
||||
if you also proxy x86_64 Arch.
|
||||
|
||||
Arch Linux ARM mirrors use `$arch/$repo` instead of `$repo/os/$arch`. In
|
||||
`/etc/pacman.d/mirrorlist`:
|
||||
|
||||
```
|
||||
Server = http://localhost:8080/arch/archlinuxarm/$arch/$repo
|
||||
```
|
||||
|
||||
Avoid `mirror.archlinuxarm.org` as the upstream. It redirects each request to a
|
||||
changing regional mirror over plain HTTP, so the database and the packages can come
|
||||
from mirrors with a different sync state.
|
||||
|
||||
### FreeBSD / pkg
|
||||
|
||||
Override the repositories in `/usr/local/etc/pkg/repos/FreeBSD.conf`. Settings
|
||||
there are merged with `/etc/pkg/FreeBSD.conf`, so only the URL and mirror type
|
||||
change and the signature settings stay. The default repository name `freebsd`
|
||||
proxies the official mirror (`https://pkg.freebsd.org`). On FreeBSD 15:
|
||||
|
||||
```
|
||||
FreeBSD-ports: {
|
||||
url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly",
|
||||
mirror_type: "none"
|
||||
}
|
||||
FreeBSD-ports-kmods: {
|
||||
url: "http://localhost:8080/freebsd/freebsd/${ABI}/kmods_quarterly_${VERSION_MINOR}",
|
||||
mirror_type: "none"
|
||||
}
|
||||
FreeBSD-base: {
|
||||
url: "http://localhost:8080/freebsd/freebsd/${ABI}/base_release_${VERSION_MINOR}",
|
||||
mirror_type: "none"
|
||||
}
|
||||
```
|
||||
|
||||
FreeBSD 14 has a single repository called `FreeBSD`:
|
||||
|
||||
```
|
||||
FreeBSD: {
|
||||
url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly",
|
||||
mirror_type: "none"
|
||||
}
|
||||
```
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
pkg update
|
||||
```
|
||||
|
||||
Keep the path after `${ABI}` that `pkg -vv` shows for each repository
|
||||
(`quarterly`, `latest`, `base_latest`, ...) and replace only the host part.
|
||||
|
||||
The repository catalogues (`meta.conf`, `packagesite.pkg`, `data.pkg`) carry
|
||||
their signature inside the archive and are served byte-for-byte unchanged
|
||||
through the metadata cache (`metadata_ttl`, stale fallback). Enable
|
||||
`cache_metadata` so `pkg update` keeps working while the upstream is down.
|
||||
Packages under `Hashed/` have their content hash in the file name, so they are
|
||||
stored in the shared artifact cache as `pkg:bsd/freebsd/{name}@{version}` and
|
||||
remain available when the upstream is unreachable.
|
||||
|
||||
Other files are passed through uncached. That includes the bootstrap package
|
||||
under `Latest/`, whose signature file is rewritten with every repository build,
|
||||
and package files without a hash in their name, which can be rebuilt under the
|
||||
same name. The denylist still applies to unhashed package files. Since only
|
||||
cached artifacts can be scanned, both kinds of package file are refused while
|
||||
a scanner covers `freebsd`; use repositories built with `pkg repo --hash` and
|
||||
bootstrap pkg without the proxy.
|
||||
|
||||
### GitHub Releases / mise (aqua backend)
|
||||
|
||||
Configure named generic upstreams:
|
||||
|
|
@ -937,72 +606,8 @@ Release assets are cached permanently after the first download and keep
|
|||
installing while GitHub is down. Tag lookups through `api.github.com` are
|
||||
cached for `metadata_ttl` and served stale during an outage or rate limit.
|
||||
Commit a `mise.lock` and install with `mise install --locked` so pinned
|
||||
installs need no API call at all.
|
||||
|
||||
If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for
|
||||
`https://api.github.com` under `upstream.auth`. Everyone who can reach the
|
||||
proxy can then make requests with that token, so the proxy requires
|
||||
`generic_paths` listing the API paths it may serve, and the token should be a
|
||||
fine-grained one with read-only access to public repositories:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
generic_paths:
|
||||
github-api:
|
||||
- "repos/*/*/releases"
|
||||
- "repos/*/*/releases/**"
|
||||
- "repos/*/*/tags"
|
||||
auth:
|
||||
"https://api.github.com":
|
||||
type: bearer
|
||||
token: "${GITHUB_TOKEN}"
|
||||
```
|
||||
|
||||
Maven, mvnd, Node and Go downloads are cached the same way once their hosts
|
||||
are configured as generic upstreams; their version-pinned paths are built in
|
||||
(see [configuration](docs/configuration.md)):
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
generic:
|
||||
apache: "https://archive.apache.org"
|
||||
nodejs: "https://nodejs.org"
|
||||
golang: "https://dl.google.com/go"
|
||||
```
|
||||
|
||||
```toml
|
||||
[settings]
|
||||
node.mirror_url = "http://localhost:8080/generic/nodejs/dist/"
|
||||
go.download_mirror = "http://localhost:8080/generic/golang"
|
||||
|
||||
[settings.url_replacements]
|
||||
"https://archive.apache.org/" = "http://localhost:8080/generic/apache/"
|
||||
```
|
||||
|
||||
Other download hosts need their version-pinned paths declared in
|
||||
`upstream.generic_artifacts`.
|
||||
|
||||
### Git repositories
|
||||
|
||||
Configure named Git servers:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
git:
|
||||
github: "https://github.com"
|
||||
```
|
||||
|
||||
Then point clones at the proxy:
|
||||
|
||||
```bash
|
||||
git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/
|
||||
```
|
||||
|
||||
Each repository is cached as a bare clone and refreshed whenever its branches
|
||||
or tags change upstream. Git LFS objects are cached in the artifact cache;
|
||||
git-lfs uses the proxy for them with no further setup. See [configuration](docs/configuration.md#git-repositories)
|
||||
for setting the rewrite through environment variables, size limits, private
|
||||
repositories and outage behavior.
|
||||
installs need no API call at all. Add a bearer token for `https://api.github.com`
|
||||
under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
|
@ -1054,8 +659,6 @@ base_url: "http://localhost:8080"
|
|||
storage:
|
||||
url: "file:///var/cache/proxy/artifacts"
|
||||
max_size: "10GB" # Optional: evict LRU when exceeded
|
||||
retention:
|
||||
default: "30d" # Optional: evict unused artifacts after 30 days; excludes generic
|
||||
|
||||
database:
|
||||
driver: "sqlite"
|
||||
|
|
@ -1089,7 +692,7 @@ Run with config file:
|
|||
|
||||
### PostgreSQL
|
||||
|
||||
SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres:
|
||||
SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
|
|
@ -1197,9 +800,6 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
|
|||
# Mirror all versions of a package
|
||||
proxy mirror pkg:npm/lodash
|
||||
|
||||
# Mirror only the newest version of a package
|
||||
proxy mirror --latest pkg:npm/lodash
|
||||
|
||||
# Mirror from a CycloneDX or SPDX SBOM
|
||||
proxy mirror --sbom sbom.cdx.json
|
||||
|
||||
|
|
@ -1208,17 +808,9 @@ proxy mirror --dry-run pkg:npm/lodash
|
|||
|
||||
# Control parallelism
|
||||
proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
|
||||
|
||||
# Mirror a container image for one platform
|
||||
proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
|
||||
|
||||
# Mirror a commit of an upstream.git repository into the git cache
|
||||
proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
|
||||
```
|
||||
|
||||
The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror.
|
||||
|
||||
With `scanning.enabled`, the mirror command refuses every package and container image so nothing is cached unscanned. Each one is reported as failed, so the command exits with status 1. Scanners fetch the staged bytes from the running server, which this command does not start. Use the mirror API below instead, since its jobs run inside the server and go through the pre-cache scan. `--dry-run` still lists what would be mirrored.
|
||||
The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped.
|
||||
|
||||
A mirror API is also available when the server is running:
|
||||
|
||||
|
|
@ -1240,18 +832,6 @@ curl http://localhost:8080/api/mirror/mirror-1
|
|||
curl -X DELETE http://localhost:8080/api/mirror/mirror-1
|
||||
```
|
||||
|
||||
### hosted
|
||||
|
||||
Push package archives to a running proxy with `hosted.enabled`, list them and delete them. See [Hosted Packages](docs/configuration.md#hosted-packages).
|
||||
|
||||
```bash
|
||||
export PROXY_URL=https://proxy.example.com PROXY_HOSTED_TOKEN=...
|
||||
proxy hosted push npm request 2.88.3 ./request-2.88.3.tgz
|
||||
proxy hosted push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip
|
||||
proxy hosted list npm request
|
||||
proxy hosted delete npm request 2.88.3
|
||||
```
|
||||
|
||||
### stats
|
||||
|
||||
Show cache statistics without running the server.
|
||||
|
|
@ -1312,9 +892,6 @@ Recently cached:
|
|||
| `GET /npm/*` | npm registry protocol |
|
||||
| `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream |
|
||||
| `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream |
|
||||
| `GET /npm/-/package/{name}/dist-tags` | `npm dist-tag ls`, answered from the packument the proxy serves (cooldown and pushed versions applied); changing tags is not supported |
|
||||
| `GET /npm/-/npm/v1/attestations/{name}@{version}` | npm provenance attestations of a version, fetched from the package's registry and cached |
|
||||
| `GET /npm/-/v1/search`, `GET /npm/-/ping` | `npm search` and `npm ping`, passed through to upstream uncached |
|
||||
| `GET /cargo/*` | Cargo sparse index protocol |
|
||||
| `GET /gem/*` | RubyGems protocol |
|
||||
| `GET /go/*` | Go module proxy protocol |
|
||||
|
|
@ -1324,10 +901,7 @@ Recently cached:
|
|||
| `GET /maven/*` | Maven repository protocol |
|
||||
| `GET /nuget/*` | NuGet V3 API |
|
||||
| `GET /composer/*` | Composer/Packagist protocol |
|
||||
| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names |
|
||||
| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached |
|
||||
| `GET /conan/*` | Conan C/C++ protocol |
|
||||
| `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) |
|
||||
| `GET /conda/*` | Conda/Anaconda protocol |
|
||||
| `GET /cran/*` | CRAN (R) protocol |
|
||||
| `GET /julia/*` | Julia Pkg server protocol |
|
||||
|
|
@ -1337,10 +911,7 @@ Recently cached:
|
|||
| `GET /v2/*` | OCI/Docker registry protocol |
|
||||
| `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR |
|
||||
| `GET /apk/{repository}/*` | Alpine APK repository protocol |
|
||||
| `GET /arch/{repository}/*` | Arch Linux pacman repository protocol |
|
||||
| `GET /freebsd/{repository}/*` | FreeBSD pkg repository protocol |
|
||||
| `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) |
|
||||
| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only |
|
||||
| `GET /debian/*` | Debian/APT repository protocol (main archive) |
|
||||
| `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) |
|
||||
| `GET /rpm/*` | RPM/Yum repository protocol |
|
||||
|
|
@ -1353,35 +924,6 @@ Recently cached:
|
|||
| `GET /api/mirror/{id}` | Get job status and progress |
|
||||
| `DELETE /api/mirror/{id}` | Cancel a running job |
|
||||
|
||||
### Hosted API
|
||||
|
||||
Registered when `hosted.enabled` is set. Pushing and deleting require `Authorization: Bearer <hosted.token>`.
|
||||
|
||||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `PUT /api/hosted/{ecosystem}/{name}/{version}` | Push the request body as the archive for a version (npm and composer) |
|
||||
| `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package |
|
||||
| `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version |
|
||||
|
||||
### Projects API
|
||||
|
||||
Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer <projects.token>`. See [Project inventory](docs/configuration.md#project-inventory).
|
||||
|
||||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists |
|
||||
| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version |
|
||||
| `GET /api/projects/{project}` | A project's recorded package versions |
|
||||
| `DELETE /api/projects/{project}` | Delete a project |
|
||||
|
||||
### Cache API
|
||||
|
||||
Registered when `cache_delete` is set. It has no authentication of its own.
|
||||
|
||||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `DELETE /api/cache/{ecosystem}/{name}/{version}` | Delete a version's cached files, or only `?filename=` |
|
||||
|
||||
### Enrichment API
|
||||
|
||||
The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection.
|
||||
|
|
@ -1535,7 +1077,9 @@ Response:
|
|||
{
|
||||
"cached_artifacts": 142,
|
||||
"total_size_bytes": 523456789,
|
||||
"total_size": "499.2 MB"
|
||||
"total_size": "499.2 MB",
|
||||
"storage_url": "file:///path/to/cache/artifacts",
|
||||
"database_path": "./cache/proxy.db"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -1567,7 +1111,6 @@ Response:
|
|||
The proxy serves a web UI under `/ui`. No separate frontend build is needed -- templates and assets are embedded in the binary. `GET /` redirects to `/ui/`. The UI is mounted under its own prefix so a reverse proxy can apply different access rules to it than to the package endpoints (for example, requiring auth for `PathPrefix(/ui)` while leaving `/npm`, `/pypi` etc. open to build machines).
|
||||
|
||||
- **Dashboard** (`/ui/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview.
|
||||
- **Analytics** (`/ui/analytics`) -- accumulated download size as a ring broken down by ecosystem with the total in the middle, the cache size, artifact, package and version counts, a per-ecosystem table, the vulnerability overview, and a Runtime card mirroring every counter `/metrics` exposes. See [Analytics](#analytics).
|
||||
- **Install guide** (`/ui/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here.
|
||||
- **Package browser** (`/ui/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count.
|
||||
- **Search** (`/ui/search?q=...`) -- search cached packages by name.
|
||||
|
|
@ -1575,7 +1118,6 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t
|
|||
- **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts.
|
||||
- **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews.
|
||||
- **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files.
|
||||
- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version.
|
||||
|
||||
## Monitoring
|
||||
|
||||
|
|
@ -1585,17 +1127,10 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
|||
|--------|------|--------|-------------|
|
||||
| `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status |
|
||||
| `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration |
|
||||
| `proxy_cache_hits_total` | counter | `ecosystem`, `kind` | Cache hits. `kind` is `artifact` or `metadata`. |
|
||||
| `proxy_cache_misses_total` | counter | `ecosystem`, `kind` | Cache misses. `kind` is `artifact` or `metadata`; metadata misses are counted only while `cache_metadata` is on. |
|
||||
| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits |
|
||||
| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses |
|
||||
| `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts |
|
||||
| `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts |
|
||||
| `proxy_metadata_cache_served_bytes_total` | counter | `ecosystem` | Metadata bytes served from the metadata cache. |
|
||||
| `proxy_metadata_cache_size_bytes` | gauge | `ecosystem` | Size of cached metadata documents per ecosystem. |
|
||||
| `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. |
|
||||
| `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) |
|
||||
| `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit |
|
||||
| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them |
|
||||
| `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them |
|
||||
| `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream |
|
||||
| `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures |
|
||||
| `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
|
||||
|
|
@ -1604,20 +1139,8 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
|||
| `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). |
|
||||
| `proxy_circuit_breaker_state` | gauge | `registry` | Artifact-fetch circuit breaker state per upstream registry (0 closed, 2 open). Published once that registry's breaker has tripped. |
|
||||
| `proxy_circuit_breaker_trips_total` | counter | `registry` | Circuit breaker trips per upstream registry. |
|
||||
| `proxy_upstream_rate_limit_holds_total` | counter | `host` | Holds placed on an upstream host after it rate limited the proxy with HTTP 429, or 503 with a `Retry-After`. See `upstream_holds` under [Health Check](#health-check). |
|
||||
| `proxy_ecosystem_downloaded_bytes` | gauge | `ecosystem` | Accumulated bytes served from cache: cache hits multiplied by the artifact size they served. |
|
||||
| `proxy_ecosystem_artifact_downloads` | gauge | `ecosystem` | Accumulated artifact downloads served from cache. |
|
||||
| `proxy_ecosystem_cache_size_bytes` | gauge | `ecosystem` | Size of cached artifacts per ecosystem. |
|
||||
| `proxy_ecosystem_cached_artifacts` | gauge | `ecosystem` | Number of cached artifacts per ecosystem. |
|
||||
| `proxy_ecosystem_packages` | gauge | `ecosystem` | Known packages per ecosystem. |
|
||||
| `proxy_ecosystem_versions` | gauge | `ecosystem` | Known package versions per ecosystem. |
|
||||
| `proxy_response_bytes_total` | counter | `ecosystem` | Response body bytes written to clients. Route-labelled, see the label caveat below. |
|
||||
| `proxy_client_requests_total` | counter | `client` | Requests by client tool, from the User-Agent. |
|
||||
| `proxy_client_response_bytes_total` | counter | `client` | Response bytes by client tool. |
|
||||
|
||||
The `ecosystem` label on `proxy_requests_total` and `proxy_request_duration_seconds` is the mounted route a request arrived on, not the ecosystem recorded against the package it served: `/gem` reports as `rubygems`, `/go` as `golang`, `/composer` as `packagist`, `/apk` as `alpine` and `/v2` as `oci`. Anything outside a package route -- the UI, `/health`, `/metrics`, `/stats` -- reports as `other`, and so did `/apk`, `/helm`, `/homebrew`, `/generic` and `/swift` before they were listed; traffic on those five routes now appears under its own name instead.
|
||||
|
||||
Cache size, artifact count and the per-ecosystem gauges are refreshed every 60 seconds, from a single pass over the database. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request.
|
||||
Cache size and artifact count are refreshed every 60 seconds. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request.
|
||||
|
||||
The breaker metrics carry one series per upstream host, but only for hosts whose breaker has tripped at least once since startup. A breaker is created per host the proxy fetches artifacts from, and for some ecosystems that host comes from upstream metadata rather than from configuration (composer takes it from a package's `dist.url`, helm from the chart URLs in `index.yaml`), so publishing every host would let upstream content grow the series count for the lifetime of the process. Once a host has tripped it keeps reporting, so a recovery still shows up as a transition to 0 rather than as a series that vanishes. `/health` is not a persistent time series and lists every breaker, tripped or not.
|
||||
|
||||
|
|
@ -1625,82 +1148,6 @@ The `registry` label is the host of the URL the artifact was fetched from. Becau
|
|||
|
||||
Alert on `proxy_circuit_breaker_state == 2` sustained for more than a few minutes: while a breaker is open, artifact downloads for that upstream fail with HTTP 502 on every cache miss, and only a single probe request per backoff interval reaches the upstream. Cached artifacts keep serving, and so does metadata for the same ecosystem (metadata does not go through the circuit breaker), so installs fail in a way that looks like a partial upstream outage.
|
||||
|
||||
#### Accumulated download size
|
||||
|
||||
`proxy_ecosystem_downloaded_bytes` is, for every cached artifact, the number of times it was served multiplied by its size. It answers "how much traffic has this proxy actually carried", which is the number that matters when sizing egress or justifying the cache. Two properties are worth knowing before alerting on it.
|
||||
|
||||
**It counts cache hits, not upstream fetches.** The request that first pulls an artifact through the proxy is a miss and is not counted; only later hits are. So the accumulated total is also the upstream bandwidth the cache has saved, not the total bytes the proxy has ever sent.
|
||||
|
||||
**Eviction removes history.** Evicting an artifact clears its size, so its past hits drop out of the total. That is why these are gauges rather than counters, and why the figure can step downwards. Chart them with `max_over_time` rather than `increase`, and read a drop after an eviction sweep as expected rather than as data loss.
|
||||
|
||||
Nothing at the schema level ties `artifacts.version_purl` to a version row, so a cached artifact can end up with no ecosystem to attribute it to. Those are reported under the ecosystem `unattributed` rather than dropped, which keeps the per-ecosystem figures adding up to `proxy_cache_size_bytes` and `proxy_cached_artifacts_total`. A non-zero `unattributed` means the database holds artifact rows whose version or package rows have gone missing.
|
||||
|
||||
The `ecosystem` label on these six is taken from the package record and normalized, so aliases collapse: a database carrying both `gem` and `rubygems` rows -- the proxy writes the former, git-pkgs the latter -- reports one `rubygems` series with the two summed.
|
||||
|
||||
### Analytics
|
||||
|
||||
`/ui/analytics` reports the accumulated download size as a ring broken down by ecosystem, the cache figures from the dashboard, a per-ecosystem table, the vulnerability overview, and a **Runtime** card covering every remaining metric `/metrics` exposes.
|
||||
|
||||
The ring shows at most six slices, because part-to-whole stops being readable past that. When more ecosystems are active the smallest are folded into a single "Other" slice; the table below lists every one of them, so nothing is hidden, only summarised.
|
||||
|
||||
#### No history is kept
|
||||
|
||||
The proxy stores no time series. The page reads the database and the in-process metric registry at request time and reports current state; there is nowhere for it to read yesterday's figures from, and nothing is written for tomorrow. That splits the figures in two, and the page says which is which.
|
||||
|
||||
**Database-derived figures survive a restart.** Download volume, cache size and the package, version and artifact counts come from the `artifacts`, `packages` and `versions` tables, so they are as durable as the database.
|
||||
|
||||
**Registry-derived figures do not.** Everything in the Runtime card -- request counts and latencies, cache hit rate, upstream and storage errors, circuit breaker state, scan results -- lives only in this process's Prometheus registry and starts from zero on restart. A small number there next to a large one above just means the proxy started recently.
|
||||
|
||||
For history, trends and alerting, scrape `/metrics` with Prometheus. That is the intended split: the UI answers "what is true now", Prometheus answers "what happened".
|
||||
|
||||
The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way.
|
||||
|
||||
#### Artifact and metadata caches
|
||||
|
||||
The proxy keeps two caches. Artifacts (tarballs, wheels, crates and so on) live in storage and are what `proxy_cache_size_bytes`, `proxy_ecosystem_downloaded_bytes` and the other artifact figures describe. Registry metadata is cached separately when `cache_metadata` is on. It is stored persistently in the `metadata_cache` table rather than held in memory.
|
||||
|
||||
The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keeps the two apart, so an artifact hit rate is `sum(rate(proxy_cache_hits_total{kind="artifact"}[5m]))` over the same expression with misses added. Metadata has its own size and entry gauges plus `proxy_metadata_cache_served_bytes_total`. That one is a process counter rather than a database total, because metadata hits are not recorded per row. On `/ui/analytics` metadata appears in its own card and in the Runtime card's metadata lookups, never mixed into the artifact figures. The card and the two gauges read one 60-second snapshot, like the artifact figures. The bundled Grafana dashboard's hit ratio covers artifacts only, while its hits and misses panel shows each kind as its own series.
|
||||
|
||||
#### Three ecosystem label sets
|
||||
|
||||
`ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which.
|
||||
|
||||
**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
|
||||
|
||||
**From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all.
|
||||
|
||||
**From the handler's own name.** `proxy_upstream_fetch_duration_seconds` and `proxy_upstream_errors_total`, which report `composer`, `gem` and `go` where the other two sets report `packagist`, `rubygems` and `golang`. These are published series and are deliberately left as they are; renaming them would break existing queries and alerts.
|
||||
|
||||
### Request sources
|
||||
|
||||
Package managers do not say who invoked them. A request from `pip` or `go` carries a `Host`, an `Accept` and a `User-Agent` -- no `Referer`, no originating URL, nothing naming a repository, pipeline or job. Whatever identity you want has to come from something on the wire, so the proxy attributes requests by the two things always present.
|
||||
|
||||
**Address** -- the TCP peer, or the leftmost `X-Forwarded-For` entry when `trust_forwarded_for` is enabled. Enable that only behind a load balancer or ingress that sets the header; any client can send it, so in front of one it lets a caller forge its own attribution and, by cycling synthetic addresses, fill the table and push every genuine caller into the overflow row. The totals stay correct; the attribution is what is lost.
|
||||
|
||||
**Client** -- the tool, taken from the leading User-Agent token: `pip`, `npm`, `go`, `docker`, `apt`, `curl` and so on. Anything unrecognised reports as `other`.
|
||||
|
||||
Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtime -> Request sources**, as a table of the busiest callers by bytes downloaded plus a per-tool breakdown. The table is in-memory and process-lifetime, like the rest of that card. It tracks 200 callers, evicting the least recently seen once full, and summarises everything it is not showing individually -- both evicted callers and those ranked below the display limit -- in a single "other callers" row, so the rows always add up to the totals above them.
|
||||
|
||||
**The flag defaults to off because the page is not authenticated.** `/ui` carries no auth of its own -- it is mounted under its own prefix so a reverse proxy *can* gate it separately, as [Behind a Reverse Proxy](#behind-a-reverse-proxy) describes, but nothing makes you -- and until now it exposed only package data. The sources table changes what is on offer: anyone who can reach the proxy can read the addresses of your build fleet, which tool each runs, and how much each pulled. Turn it on once `/ui` is gated, or leave it off and read the same detail from the access log.
|
||||
|
||||
**What this can and cannot tell you.** How much an address gives you depends entirely on your network. A fleet of build machines with stable addresses attributes cleanly. Containerised CI usually does not: with Docker or Kubernetes executors every job gets an ephemeral address, and egress is commonly NAT'd behind one gateway, so you get runner-node or gateway granularity, not per-project. If you need per-project attribution the caller has to send something naming itself -- a basic-auth username, or a per-project base URL -- which the proxy does not currently read. Say so and it can be added.
|
||||
|
||||
**Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to.
|
||||
|
||||
### Failure details
|
||||
|
||||
The failure counts under **Runtime** on `/ui/analytics` -- upstream errors, storage errors, integrity failures, health probe failures and scan errors -- say how often something failed, not what. Set `ui_failure_details: true` and each count gets a **Latest** list underneath it, newest first: the package URL and filename, the upstream URL or storage path, the error, and how long ago it happened.
|
||||
|
||||
The list keeps the last 20 failures per row, in memory only, so it starts empty when the proxy starts, like the counts above it. Credentials, query strings and fragments are stripped from every URL before it is kept, including URLs inside error messages, since that is where registry tokens and signed-URL signatures live.
|
||||
|
||||
The flag defaults to off for the same reason as `ui_request_sources`: package names and upstream hosts say what your users depend on, and `/ui` is not authenticated. Package names and URLs are never Prometheus labels; the same detail is in the proxy's log.
|
||||
|
||||
### Grafana dashboard
|
||||
|
||||
A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID.
|
||||
|
||||
It carries three ecosystem filters rather than one, because `ecosystem` means three different things across `/metrics` -- see the label sets above. **Ecosystem** filters the database-derived gauges, **Route** the request-path counters, and **Upstream** the two upstream fetch metrics. All three are query variables, so they populate from whatever labels your proxy is actually reporting; a panel is on the one its metric belongs to, and the panel descriptions say which.
|
||||
|
||||
### Health Check
|
||||
|
||||
`/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail.
|
||||
|
|
@ -1715,9 +1162,6 @@ It carries three ecosystem filters rather than one, because `ecosystem` means th
|
|||
"circuit_breakers": {
|
||||
"registry.npmjs.org": "closed",
|
||||
"static.crates.io": "open"
|
||||
},
|
||||
"upstream_holds": {
|
||||
"repo1.maven.org": "2026-10-14T11:15:29Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -1726,8 +1170,6 @@ Failing checks include an `"error"` field. Storage failures also include a `"ste
|
|||
|
||||
`circuit_breakers` reports the state of each upstream's artifact-fetch circuit breaker (`"open"` or `"closed"`), keyed by upstream host — or by the `hostless-url-<digest>` placeholder described under [Monitoring](#monitoring) where the fetch URL has no host to read. The key is omitted until the proxy has fetched an artifact from at least one upstream, and a host appears only once a breaker has been created for it. Breakers trip after repeated upstream failures and retry the upstream after an exponential backoff. While one is open, artifact downloads for that host return HTTP 502 on a cache miss without contacting the upstream; already-cached artifacts are still served from storage, since the cache is checked before the fetcher. A breaker is reported as `"open"` throughout its backoff, including the half-open window in which it admits one probe request to test recovery. Breaker state is per process and in memory, so a restart clears it, but a restart is not needed for recovery: the backoff keeps retrying for as long as the breaker is open, so it closes on its own once the upstream serves again.
|
||||
|
||||
`upstream_holds` lists upstream hosts that answered the proxy with HTTP 429, or 503 with a `Retry-After`, and the time until which the proxy sends them nothing. Every request to a held host, metadata and artifacts alike, fails locally for the duration the upstream asked for (capped at one hour), or for one minute doubling up to fifteen when the upstream gave no `Retry-After`. Cached metadata is served stale during a hold, and anything uncached returns HTTP 503 with the remaining `Retry-After` so a client can pace itself too. Metadata requests do not go through the circuit breaker, so without the hold a rate-limited upstream would keep receiving one request per cache miss, which is exactly the traffic it is asking the proxy to stop sending. The first non-limited response after a hold ends clears it, and `proxy_upstream_rate_limit_holds_total` counts the holds placed. Holds are recorded in the database, so instances sharing one honour each other's holds and list them here too. The key is omitted while no host is held.
|
||||
|
||||
An open breaker does **not** set `status` to `"error"` or change the HTTP status code: it reports a specific upstream refusing to serve, not this proxy being unfit to receive traffic, and failing the readiness probe over one unhealthy upstream would pull the pod out of rotation for every other ecosystem too. Use `proxy_circuit_breaker_state` for alerting on it.
|
||||
|
||||
Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it.
|
||||
|
|
@ -1772,23 +1214,13 @@ sudo systemctl start proxy
|
|||
|
||||
### Docker
|
||||
|
||||
Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`:
|
||||
|
||||
```bash
|
||||
docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest
|
||||
```
|
||||
|
||||
To build the image from the repo instead:
|
||||
A Dockerfile is included in the repo. Build and run:
|
||||
|
||||
```bash
|
||||
docker build -t proxy .
|
||||
docker run -p 8080:8080 -v proxy-data:/data proxy
|
||||
```
|
||||
|
||||
The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`.
|
||||
|
||||
With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them.
|
||||
|
||||
With Postgres and S3:
|
||||
|
||||
```bash
|
||||
|
|
@ -1878,12 +1310,6 @@ cache/artifacts/
|
|||
└── nginx-1.24.0-1.fc39.x86_64.rpm
|
||||
```
|
||||
|
||||
To delete one version's cached files, so the next request fetches them again, enable `cache_delete` and use the Delete buttons on the version page in the web UI, or the [cache API](docs/configuration.md#cache-delete-api):
|
||||
|
||||
```bash
|
||||
curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar'
|
||||
```
|
||||
|
||||
Cache metadata is stored in SQLite (default) or PostgreSQL. To clear a local cache:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -1,278 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/handler"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultProxyURL = "http://localhost:8080"
|
||||
hostedTokenEnv = "PROXY_HOSTED_TOKEN"
|
||||
hostedClientTimeout = 10 * time.Minute
|
||||
|
||||
pushArgCount = 4 // ecosystem, name, version, file
|
||||
listArgCount = 2 // ecosystem, name
|
||||
deleteArgCount = 3 // ecosystem, name, version
|
||||
deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files
|
||||
tablePadding = 2
|
||||
)
|
||||
|
||||
// hostedArtifact mirrors the JSON the /api/hosted endpoints return.
|
||||
type hostedArtifact struct {
|
||||
Version string `json:"version"`
|
||||
Filename string `json:"filename"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
PushedAt time.Time `json:"pushed_at"`
|
||||
}
|
||||
|
||||
type hostedPushResponse struct {
|
||||
Ecosystem string `json:"ecosystem"`
|
||||
Name string `json:"name"`
|
||||
hostedArtifact
|
||||
}
|
||||
|
||||
type hostedListResponse struct {
|
||||
Ecosystem string `json:"ecosystem"`
|
||||
Name string `json:"name"`
|
||||
Artifacts []hostedArtifact `json:"artifacts"`
|
||||
}
|
||||
|
||||
type hostedCommand struct {
|
||||
flags *flag.FlagSet
|
||||
proxyURL *string
|
||||
usage string
|
||||
}
|
||||
|
||||
func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand {
|
||||
fs := flag.NewFlagSet(name, flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
proxyURL := getenv("PROXY_URL")
|
||||
if proxyURL == "" {
|
||||
proxyURL = defaultProxyURL
|
||||
}
|
||||
cmd := hostedCommand{flags: fs, usage: usage}
|
||||
cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage)
|
||||
fs.SetOutput(os.Stderr)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
// parse parses flags and checks for between minArgs and maxArgs positional
|
||||
// arguments.
|
||||
func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) {
|
||||
if err := c.flags.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
c.flags.Usage()
|
||||
}
|
||||
return nil, fmt.Errorf("%w\nusage: %s", err, c.usage)
|
||||
}
|
||||
if n := c.flags.NArg(); n < minArgs || n > maxArgs {
|
||||
return nil, fmt.Errorf("usage: %s", c.usage)
|
||||
}
|
||||
return c.flags.Args(), nil
|
||||
}
|
||||
|
||||
func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string {
|
||||
segments := []string{url.PathEscape(ecosystem)}
|
||||
for segment := range strings.SplitSeq(name, "/") {
|
||||
segments = append(segments, url.PathEscape(segment))
|
||||
}
|
||||
for _, v := range version {
|
||||
segments = append(segments, url.PathEscape(v))
|
||||
}
|
||||
return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
func hostedToken(getenv func(string) string) (string, error) {
|
||||
token := getenv(hostedTokenEnv)
|
||||
if token == "" {
|
||||
return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv)
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("push", "proxy hosted push [-url URL] <ecosystem> <name> <version> <file>",
|
||||
"Push a package archive to the proxy", getenv)
|
||||
positional, err := cmd.parse(args, pushArgCount, pushArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, err := hostedToken(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3]
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
// The proxy checks the upload against this, so a truncated or corrupted
|
||||
// transfer is refused rather than stored.
|
||||
digest := sha256.New()
|
||||
size, err := io.Copy(digest, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
segments := []string{version}
|
||||
if handler.HostedFileInPath(ecosystem) {
|
||||
segments = append(segments, filepath.Base(path))
|
||||
}
|
||||
endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil))
|
||||
req, err := http.NewRequest(http.MethodPut, endpoint, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
// The proxy refuses an existing file or a bad token before reading the
|
||||
// body, so the file is only uploaded once it would be accepted.
|
||||
req.Header.Set("Expect", "100-continue")
|
||||
|
||||
var pushed hostedPushResponse
|
||||
if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n",
|
||||
pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256)
|
||||
return err
|
||||
}
|
||||
|
||||
func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("list", "proxy hosted list [-url URL] <ecosystem> <name>", "List pushed package archives", getenv)
|
||||
positional, err := cmd.parse(args, listArgCount, listArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var listed hostedListResponse
|
||||
if err := doHosted(client, req, http.StatusOK, &listed); err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0)
|
||||
_, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED")
|
||||
for _, a := range listed.Artifacts {
|
||||
_, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339))
|
||||
}
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] <ecosystem> <name> <version> [file]",
|
||||
"Delete a pushed version, or one generic file of it", getenv)
|
||||
positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) {
|
||||
return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage)
|
||||
}
|
||||
token, err := hostedToken(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if err := doHosted(client, req, http.StatusNoContent, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
deleted := positional[1] + "@" + positional[2]
|
||||
if len(positional) > deleteArgCount {
|
||||
deleted += " " + positional[deleteArgCount]
|
||||
}
|
||||
_, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted)
|
||||
return err
|
||||
}
|
||||
|
||||
// doHosted sends req and decodes a successful response into out. Any other
|
||||
// status is returned as an error carrying the server's message.
|
||||
func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error {
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != wantStatus {
|
||||
var apiErr struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" {
|
||||
return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status)
|
||||
}
|
||||
return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
const hostedUsage = "usage: proxy hosted <push|list|delete> [flags] ...\n\n" +
|
||||
" push Push a package archive to the proxy\n" +
|
||||
" list List pushed package archives\n" +
|
||||
" delete Delete a pushed version, or one generic file of it"
|
||||
|
||||
type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error
|
||||
|
||||
var hostedSubcommands = map[string]hostedSubcommand{
|
||||
"push": runPush,
|
||||
"list": runList,
|
||||
"delete": runDelete,
|
||||
}
|
||||
|
||||
// runHosted runs the hosted subcommand named by args[0].
|
||||
func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(hostedUsage)
|
||||
}
|
||||
run, ok := hostedSubcommands[args[0]]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage)
|
||||
}
|
||||
return run(args[1:], getenv, stdout, client)
|
||||
}
|
||||
|
||||
// runHostedCommand runs proxy hosted against the proxy's /api/hosted
|
||||
// endpoints and exits non-zero on failure.
|
||||
func runHostedCommand() {
|
||||
client := &http.Client{Timeout: hostedClientTimeout}
|
||||
if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recordedRequest struct {
|
||||
method, path, query, authorization, body, expect string
|
||||
contentLength int64
|
||||
}
|
||||
|
||||
func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) {
|
||||
t.Helper()
|
||||
recorded := &recordedRequest{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
*recorded = recordedRequest{
|
||||
method: r.Method,
|
||||
path: r.URL.EscapedPath(),
|
||||
query: r.URL.RawQuery,
|
||||
authorization: r.Header.Get("Authorization"),
|
||||
body: string(body),
|
||||
expect: r.Header.Get("Expect"),
|
||||
contentLength: r.ContentLength,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, response)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, recorded
|
||||
}
|
||||
|
||||
func hostedEnv(vars map[string]string) func(string) string {
|
||||
return func(key string) string { return vars[key] }
|
||||
}
|
||||
|
||||
func writeArchive(t *testing.T, name, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("writing archive: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestRunPush(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated,
|
||||
`{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`)
|
||||
archive := writeArchive(t, "node-20.1.0.tgz", "tarball")
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" {
|
||||
t.Errorf("request = %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
// sha256 of "tarball", so the proxy can check the upload.
|
||||
const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0"
|
||||
if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 {
|
||||
t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query)
|
||||
}
|
||||
if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) {
|
||||
t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength)
|
||||
}
|
||||
if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") {
|
||||
t.Errorf("output = %q, want filename and sha256", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushGenericNamesTheFile(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated,
|
||||
`{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`)
|
||||
archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball")
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
|
||||
t.Errorf("path = %q, want the file name appended", recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushErrors(t *testing.T) {
|
||||
srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`)
|
||||
archive := writeArchive(t, "serde-1.0.200.crate", "crate")
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") {
|
||||
t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err)
|
||||
}
|
||||
|
||||
err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") {
|
||||
t.Errorf("rejected push error = %v, want the server's message", err)
|
||||
}
|
||||
|
||||
err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "usage") {
|
||||
t.Errorf("push with missing arguments error = %v, want usage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushUsesProxyURLFromEnv(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`)
|
||||
archive := writeArchive(t, "request-2.88.3.tgz", "tarball")
|
||||
|
||||
err := runPush([]string{"npm", "request", "2.88.3", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/npm/request/2.88.3" {
|
||||
t.Errorf("path = %q", recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunList(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusOK,
|
||||
`{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`)
|
||||
var stdout bytes.Buffer
|
||||
|
||||
if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil {
|
||||
t.Fatalf("runList: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" {
|
||||
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
|
||||
}
|
||||
if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") {
|
||||
t.Errorf("output = %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDelete(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runDelete: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" {
|
||||
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
|
||||
}
|
||||
|
||||
missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`)
|
||||
err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") {
|
||||
t.Errorf("delete of a missing version error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeleteOneFile(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runDelete: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
|
||||
t.Errorf("path = %q", recorded.path)
|
||||
}
|
||||
if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" {
|
||||
t.Errorf("output = %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "only generic files") {
|
||||
t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err)
|
||||
}
|
||||
if recorded.method != "" {
|
||||
t.Errorf("refused delete sent %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHostedDispatchesSubcommands(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`)
|
||||
|
||||
if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil {
|
||||
t.Fatalf("runHosted list: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" {
|
||||
t.Errorf("request = %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
|
||||
for _, args := range [][]string{nil, {"bogus"}} {
|
||||
err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted <push|list|delete>") {
|
||||
t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) {
|
||||
err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient)
|
||||
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") {
|
||||
t.Errorf("push usage error = %v, want it to name proxy hosted push", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -5,11 +5,6 @@
|
|||
// @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing.
|
||||
// @BasePath /
|
||||
//
|
||||
// @securityDefinitions.apikey BearerAuth
|
||||
// @in header
|
||||
// @name Authorization
|
||||
// @description "Bearer " followed by hosted.token, or projects.token for /api/projects
|
||||
//
|
||||
// The proxy caches package artifacts from upstream registries (npm, cargo, etc.)
|
||||
// providing faster, more reliable access for development teams.
|
||||
//
|
||||
|
|
@ -22,7 +17,6 @@
|
|||
// serve Start the proxy server (default if no command given)
|
||||
// stats Show cache statistics
|
||||
// mirror Pre-populate cache from PURLs, SBOMs, or registries
|
||||
// hosted Push, list or delete packages on a running proxy (needs hosted.enabled)
|
||||
//
|
||||
// Serve Flags:
|
||||
//
|
||||
|
|
@ -82,21 +76,12 @@
|
|||
// PROXY_LOG_FORMAT - Log format
|
||||
// PROXY_ACCESS_LOG_PATH - JSONL access log path
|
||||
// PROXY_UPSTREAM_* - Upstream URLs and network access controls
|
||||
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Set false to allow Gradle PUT uploads (default true)
|
||||
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
|
||||
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
|
||||
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
|
||||
// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted
|
||||
// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting
|
||||
// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB")
|
||||
// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only
|
||||
// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only
|
||||
// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects)
|
||||
// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs
|
||||
// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d")
|
||||
// PROXY_URL - Proxy URL for proxy hosted
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
|
|
@ -127,11 +112,12 @@ import (
|
|||
|
||||
"github.com/git-pkgs/proxy/internal/config"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/denylist"
|
||||
"github.com/git-pkgs/proxy/internal/handler"
|
||||
"github.com/git-pkgs/proxy/internal/memlimit"
|
||||
"github.com/git-pkgs/proxy/internal/mirror"
|
||||
"github.com/git-pkgs/proxy/internal/server"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
const defaultTopN = 10
|
||||
|
|
@ -168,10 +154,6 @@ func main() {
|
|||
os.Args = append(os.Args[:1], os.Args[2:]...)
|
||||
runMirror()
|
||||
return
|
||||
case "hosted":
|
||||
os.Args = append(os.Args[:1], os.Args[2:]...)
|
||||
runHostedCommand()
|
||||
return
|
||||
case "-version", "--version":
|
||||
fmt.Printf("proxy %s (%s)\n", Version, Commit)
|
||||
os.Exit(0)
|
||||
|
|
@ -194,7 +176,6 @@ Commands:
|
|||
serve Start the proxy server (default)
|
||||
stats Show cache statistics
|
||||
mirror Pre-populate cache from PURLs, SBOMs, or registries
|
||||
hosted Push, list or delete packages on a running proxy
|
||||
|
||||
Run 'proxy <command> -help' for more information on a command.
|
||||
|
||||
|
|
@ -242,7 +223,6 @@ func runServe() {
|
|||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO_DOWNLOAD Cargo download upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GEM RubyGems upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO Go module proxy upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO_VULNDB Go vulnerability database URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX Hex repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX_API Hex API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PUB pub registry upstream URL\n")
|
||||
|
|
@ -255,7 +235,6 @@ func runServe() {
|
|||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n")
|
||||
|
|
@ -263,7 +242,7 @@ func runServe() {
|
|||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_OCI_DEFAULT Default OCI registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_DEBIAN Debian repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_RPM RPM repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Set false to allow Gradle PUT uploads (default true)\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
|
||||
|
|
@ -328,7 +307,6 @@ func runServe() {
|
|||
|
||||
// Setup logger
|
||||
logger := setupLogger(cfg.Log.Level, cfg.Log.Format)
|
||||
memlimit.Apply(logger)
|
||||
|
||||
// Create and start server
|
||||
srv, err := server.New(cfg, logger, server.BuildInfo{
|
||||
|
|
@ -443,8 +421,6 @@ func runMirror() {
|
|||
sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
|
||||
concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
|
||||
dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
|
||||
platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
|
||||
latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version")
|
||||
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
|
||||
|
|
@ -452,23 +428,13 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, "Examples:\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n")
|
||||
fmt.Fprintf(os.Stderr, "Flags:\n")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
_ = fs.Parse(os.Args[1:])
|
||||
purls := fs.Args()
|
||||
var platformList []string
|
||||
if *platforms != "" {
|
||||
platformList = []string{*platforms}
|
||||
}
|
||||
if _, err := mirror.ParsePlatforms(platformList); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Determine source
|
||||
var source mirror.Source
|
||||
|
|
@ -479,9 +445,9 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList}
|
||||
source = &mirror.SBOMSource{Data: data, Name: *sbomPath}
|
||||
case len(purls) > 0:
|
||||
source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList}
|
||||
source = &mirror.PURLSource{PURLs: purls}
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
|
||||
fs.Usage()
|
||||
|
|
@ -495,7 +461,6 @@ func runMirror() {
|
|||
os.Exit(1)
|
||||
}
|
||||
cfg.LoadFromEnv()
|
||||
printEnvWarnings(cfg)
|
||||
|
||||
if *storageURL != "" {
|
||||
cfg.Storage.URL = *storageURL
|
||||
|
|
@ -514,10 +479,6 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if !cfg.Storage.CacheArtifacts {
|
||||
fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
logger := setupLogger("info", "text")
|
||||
|
||||
|
|
@ -553,27 +514,21 @@ func runMirror() {
|
|||
os.Exit(1) //nolint:gocritic // db closed above
|
||||
}
|
||||
|
||||
// Build proxy with the same upstream clients and authentication as serve
|
||||
proxy, images := server.NewMirrorProxy(cfg, db, store, logger)
|
||||
// Build proxy (reuses same pipeline as serve)
|
||||
fetcher := fetch.NewFetcher()
|
||||
resolver := fetch.NewResolver()
|
||||
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
|
||||
proxy.Denylist, err = denylist.New(cfg.Denylist.Packages)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
fmt.Fprintf(os.Stderr, "invalid denylist: %v\n", err)
|
||||
os.Exit(1) //nolint:gocritic // db closed above
|
||||
}
|
||||
proxy.CacheMetadata = true // mirror always caches metadata
|
||||
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
||||
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
|
||||
proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize()
|
||||
// As serve does, resolve Swift source repositories in the denylist through
|
||||
// the upstream, and refuse to run when one can't be.
|
||||
if err := server.ConfigureDenylist(cfg, proxy); err != nil {
|
||||
_ = db.Close()
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1) //nolint:gocritic // db closed above
|
||||
}
|
||||
|
||||
m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency)
|
||||
switch s := source.(type) {
|
||||
case *mirror.PURLSource:
|
||||
s.Versions = m.Versions
|
||||
case *mirror.SBOMSource:
|
||||
s.Versions = m.Versions
|
||||
}
|
||||
m := mirror.New(proxy, db, store, logger, *concurrency)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
|
|
@ -596,9 +551,7 @@ func runMirror() {
|
|||
return
|
||||
}
|
||||
|
||||
closeGit := useCommandGit(cfg, proxy, m, logger)
|
||||
progress, err := m.Run(ctx, source)
|
||||
closeGit()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -613,75 +566,6 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// A partial mirror has to be visible in the exit status, or a CI step
|
||||
// that prewarms the cache passes even when every download failed.
|
||||
if code := mirrorExitCode(progress); code != 0 {
|
||||
_ = db.Close()
|
||||
os.Exit(code) //nolint:gocritic // db closed above
|
||||
}
|
||||
}
|
||||
|
||||
// newCommandMirror builds the mirror proxy mirror runs, which fetches and
|
||||
// lists versions from the configured upstreams.
|
||||
func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler,
|
||||
db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror {
|
||||
m := mirror.New(proxy, db, store, logger, workers)
|
||||
m.Images = images
|
||||
m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI)
|
||||
m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
|
||||
server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient)
|
||||
if cfg.Scanning.Enabled {
|
||||
refuseUnscannedMirroring(m)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
|
||||
// when upstream.git is configured, and returns the function that closes it.
|
||||
// Where it can't be opened, such as while the running server holds
|
||||
// git.cache_dir, those items fail with the reason and the rest are mirrored.
|
||||
func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
|
||||
git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
|
||||
if err != nil {
|
||||
m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
|
||||
return func() {}
|
||||
}
|
||||
if git == nil {
|
||||
return func() {}
|
||||
}
|
||||
m.Git = git
|
||||
return func() {
|
||||
if err := closeGit(); err != nil {
|
||||
logger.Warn("closing the git cache", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorExitCode returns the exit status for a finished mirror run: 1 when any
|
||||
// package version failed to mirror, 0 otherwise.
|
||||
func mirrorExitCode(progress *mirror.Progress) int {
|
||||
if progress != nil && progress.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Reasons the mirror command gives for refusing items while scanning is
|
||||
// enabled.
|
||||
const (
|
||||
mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
|
||||
mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
|
||||
)
|
||||
|
||||
// refuseUnscannedMirroring makes m fail every item instead of caching it.
|
||||
// Scanners fetch the staged bytes from the running server, which this command
|
||||
// does not start, so anything it cached would skip the pre-cache scan and
|
||||
// later be served from the cache unscanned. The mirror API's jobs run inside
|
||||
// the server with its scanners and are unaffected.
|
||||
func refuseUnscannedMirroring(m *mirror.Mirror) {
|
||||
m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason}
|
||||
m.RefusePackages = mirrorScanningPackagesReason
|
||||
}
|
||||
|
||||
func printStats(db *database.DB, popular, recent int, asJSON bool) error {
|
||||
|
|
@ -826,14 +710,6 @@ func formatSize(bytes int64) string {
|
|||
}
|
||||
}
|
||||
|
||||
// printEnvWarnings reports environment values LoadFromEnv ignored. serve
|
||||
// logs them from server.Start; mirror never gets there.
|
||||
func printEnvWarnings(cfg *config.Config) {
|
||||
for _, warning := range cfg.EnvWarnings() {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s\n", warning)
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(path string) (*config.Config, error) {
|
||||
if path != "" {
|
||||
return config.Load(path)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) {
|
|||
"PROXY_UPSTREAM_COMPOSER",
|
||||
"PROXY_UPSTREAM_COMPOSER_REPOSITORY",
|
||||
"PROXY_UPSTREAM_CONAN",
|
||||
"PROXY_UPSTREAM_CHEF",
|
||||
"PROXY_UPSTREAM_CONDA",
|
||||
"PROXY_UPSTREAM_CRAN",
|
||||
"PROXY_UPSTREAM_JULIA",
|
||||
|
|
@ -57,24 +56,3 @@ func TestServeHelpProcess(*testing.T) {
|
|||
os.Args = []string{"proxy", "serve", "-help"}
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMirrorLogsEnvWarnings(t *testing.T) {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$")
|
||||
// The negative size stops mirror before it opens any storage.
|
||||
cmd.Env = append(os.Environ(),
|
||||
"PROXY_TEST_MIRROR_ENV_WARNINGS=1",
|
||||
"PROXY_CACHE_METADATA=maybe",
|
||||
"PROXY_STORAGE_MAX_SIZE=-1")
|
||||
output, _ := cmd.CombinedOutput()
|
||||
if !strings.Contains(string(output), "PROXY_CACHE_METADATA") {
|
||||
t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorEnvWarningsProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" {
|
||||
return
|
||||
}
|
||||
os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"}
|
||||
main()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,132 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/gitcache"
|
||||
)
|
||||
|
||||
// gitMirrorUpstream serves team/repo, with one commit on main, from git
|
||||
// http-backend, and returns the server and the commit.
|
||||
func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
|
||||
t.Helper()
|
||||
git := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = []string{
|
||||
"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
|
||||
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
backend := ""
|
||||
for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
|
||||
if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
|
||||
backend = path
|
||||
}
|
||||
}
|
||||
if backend == "" {
|
||||
t.Skip("git http-backend is not installed")
|
||||
}
|
||||
root := t.TempDir()
|
||||
bare := filepath.Join(root, "team", "repo.git")
|
||||
git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
|
||||
work := t.TempDir()
|
||||
git("", "clone", "--quiet", bare, work)
|
||||
git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
|
||||
git(work, "push", "--quiet", "origin", "HEAD:main")
|
||||
head := git(work, "rev-parse", "HEAD")
|
||||
|
||||
server := httptest.NewServer(&cgi.Handler{
|
||||
Path: backend,
|
||||
Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
|
||||
})
|
||||
t.Cleanup(server.Close)
|
||||
return server, head
|
||||
}
|
||||
|
||||
func fileExistsForTest(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeGitMirrorConfig writes a config with upstream.git "up" at
|
||||
// upstreamURL and the git cache, storage and database under dir.
|
||||
func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"upstream:\n" +
|
||||
" allow_loopback: true\n" +
|
||||
" git:\n" +
|
||||
" up: " + upstreamURL + "\n" +
|
||||
"git:\n" +
|
||||
" cache_dir: " + filepath.Join(dir, "git") + "\n"
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestMirrorCommandCachesGitCommit(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 downloaded") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
|
||||
if len(cached) != 1 {
|
||||
t.Errorf("cached repositories %v, want team/repo.git", cached)
|
||||
}
|
||||
|
||||
output = runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 skipped") {
|
||||
t.Errorf("second run output:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
|
||||
// mirror API, while the running server holds git.cache_dir.
|
||||
func TestMirrorCommandGitCacheInUse(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
cache, err := gitcache.New(gitcache.Options{
|
||||
CacheDir: filepath.Join(dir, "git"),
|
||||
RefreshWait: time.Second,
|
||||
Client: &http.Client{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = cache.Close() }()
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, purl)
|
||||
for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
)
|
||||
|
||||
const swiftWidgetArchive = "acme widget archive"
|
||||
|
||||
// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package
|
||||
// registry does, with https://github.com/acme/widget as its source
|
||||
// repository, and counts archive downloads and identifier lookups. The
|
||||
// repository's identifiers are ids, or only acme.widget.
|
||||
func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) {
|
||||
t.Helper()
|
||||
if len(ids) == 0 {
|
||||
ids = []string{"acme.widget"}
|
||||
}
|
||||
identifiers, err := json.Marshal(map[string][]string{"identifiers": ids})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloads, lookups = new(atomic.Int64), new(atomic.Int64)
|
||||
sum := sha256.Sum256([]byte(swiftWidgetArchive))
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Version", "1")
|
||||
switch r.URL.Path {
|
||||
case "/identifiers":
|
||||
lookups.Add(1)
|
||||
if r.URL.Query().Get("url") != "https://github.com/acme/widget" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(identifiers)
|
||||
case "/acme/widget":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`)
|
||||
case "/acme/widget/1.0.0/Package.swift":
|
||||
w.Header().Set("Content-Type", "text/x-swift")
|
||||
_, _ = io.WriteString(w, "// swift-tools-version:5.9\n")
|
||||
case "/acme/widget/1.0.0":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:]))
|
||||
case "/acme/widget/1.0.0.zip":
|
||||
downloads.Add(1)
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = io.WriteString(w, swiftWidgetArchive)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, downloads, lookups
|
||||
}
|
||||
|
||||
// writeSwiftMirrorConfig writes a config using upstream as the Swift
|
||||
// registry, denying the denied package PURLs.
|
||||
func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"upstream:\n" +
|
||||
" swift: " + upstream + "\n" +
|
||||
" allow_loopback: true\n"
|
||||
if len(denied) > 0 {
|
||||
cfg += "denylist:\n packages:\n"
|
||||
for _, p := range denied {
|
||||
cfg += " - " + p + "\n"
|
||||
}
|
||||
}
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func cachedArtifacts(t *testing.T, dir string) int64 {
|
||||
t.Helper()
|
||||
db, err := database.Open(filepath.Join(dir, "proxy.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("opening database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
stats, err := db.GetCacheStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetCacheStats: %v", err)
|
||||
}
|
||||
return stats.TotalArtifacts
|
||||
}
|
||||
|
||||
// Without a denylist the fake registry mirrors the release, so the tests
|
||||
// below fail for the denylist and not for the setup.
|
||||
func TestMirrorSwiftRelease(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL)
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "1 downloaded") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 1 {
|
||||
t.Errorf("archive downloads = %d, want 1", n)
|
||||
}
|
||||
if n := cachedArtifacts(t, dir); n != 1 {
|
||||
t.Errorf("cached artifacts = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A denylist entry naming the package by its source repository blocks the
|
||||
// mirror download of the registry identity it resolves to.
|
||||
func TestMirrorDenylistSwiftSourceEntry(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
if n := cachedArtifacts(t, dir); n != 0 {
|
||||
t.Errorf("cached artifacts = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The registry may list several identities for one repository. A source
|
||||
// entry denies all of them, not only the first.
|
||||
func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget")
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A source repository entry the registry can't resolve stops the mirror
|
||||
// before it downloads anything, as it stops serve: left out, it would let
|
||||
// the version through.
|
||||
func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) {
|
||||
upstream, downloads, lookups := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") {
|
||||
t.Errorf("output does not report the unresolved entry:\n%s", output)
|
||||
}
|
||||
if strings.Contains(output, "Mirror complete") {
|
||||
t.Errorf("mirror ran:\n%s", output)
|
||||
}
|
||||
if lookups.Load() == 0 {
|
||||
t.Error("the entry was not looked up")
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,214 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/mirror"
|
||||
)
|
||||
|
||||
// mirrorTestEnv carries the mirror command's arguments into the helper
|
||||
// process, one per line.
|
||||
const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS"
|
||||
|
||||
// writeScanningMirrorConfig writes a config with scanning enabled and storage
|
||||
// and database under dir. The scanner address is never reached.
|
||||
func writeScanningMirrorConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"scanning:\n" +
|
||||
" enabled: true\n" +
|
||||
" signing_key: test-signing-key\n" +
|
||||
" scanners:\n" +
|
||||
" - name: test\n" +
|
||||
" url: http://127.0.0.1:1/scan\n"
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// runMirrorProcess runs proxy mirror with args in a helper process and
|
||||
// checks that it exits with wantCode.
|
||||
func runMirrorProcess(t *testing.T, wantCode int, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$")
|
||||
cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n"))
|
||||
output, err := cmd.CombinedOutput()
|
||||
|
||||
code := 0
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
code = exitErr.ExitCode()
|
||||
} else if err != nil {
|
||||
t.Fatalf("proxy mirror %v: %v\n%s", args, err, output)
|
||||
}
|
||||
if code != wantCode {
|
||||
t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output)
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
// With scanning enabled, proxy mirror refuses packages rather than caching
|
||||
// them unscanned: scanners fetch staged bytes from the running server, which
|
||||
// the command does not start.
|
||||
// A refused package counts as failed, so the command exits 1.
|
||||
func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeScanningMirrorConfig(t, dir)
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21")
|
||||
|
||||
for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
|
||||
db, err := database.Open(filepath.Join(dir, "proxy.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("opening database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
stats, err := db.GetCacheStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetCacheStats: %v", err)
|
||||
}
|
||||
if stats.TotalArtifacts != 0 {
|
||||
t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts)
|
||||
}
|
||||
|
||||
var stored []string
|
||||
_ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
stored = append(stored, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(stored) != 0 {
|
||||
t.Errorf("files written to storage: %v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run caches nothing, so it still lists what would be mirrored.
|
||||
func TestMirrorCommandDryRunWhileScanning(t *testing.T) {
|
||||
configPath := writeScanningMirrorConfig(t, t.TempDir())
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21")
|
||||
|
||||
if !strings.Contains(output, "Would mirror 1 package versions") {
|
||||
t.Errorf("dry run output:\n%s", output)
|
||||
}
|
||||
if strings.Contains(output, mirrorScanningPackagesReason) {
|
||||
t.Errorf("dry run reported the refusal:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorCommandSBOMComponentVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeScanningMirrorConfig(t, dir)
|
||||
sbomPath := filepath.Join(dir, "bom.json")
|
||||
data := []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`)
|
||||
if err := os.WriteFile(sbomPath, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "-sbom", sbomPath)
|
||||
for _, want := range []string{"Would mirror 1 package versions", "pkg:docker/ghcr.io/team/app@1.0"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("dry run missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefuseUnscannedMirroring(t *testing.T) {
|
||||
m := mirror.New(nil, nil, nil, nil, 1)
|
||||
refuseUnscannedMirroring(m)
|
||||
|
||||
if m.RefusePackages != mirrorScanningPackagesReason {
|
||||
t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason)
|
||||
}
|
||||
images, ok := m.Images.(mirror.UnavailableImages)
|
||||
if !ok || images.Reason != mirrorScanningImagesReason {
|
||||
t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorHelperProcess(*testing.T) {
|
||||
args := os.Getenv(mirrorTestEnv)
|
||||
if args == "" {
|
||||
return
|
||||
}
|
||||
os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...)
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMirrorExitCode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
progress *mirror.Progress
|
||||
want int
|
||||
}{
|
||||
{"nil progress", nil, 0},
|
||||
{"everything mirrored", &mirror.Progress{Total: 2, Completed: 2}, 0},
|
||||
{"everything cached", &mirror.Progress{Total: 2, Skipped: 2}, 0},
|
||||
{"one failure", &mirror.Progress{Total: 2, Completed: 1, Failed: 1}, 1},
|
||||
{"all failed", &mirror.Progress{Total: 3, Failed: 3}, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := mirrorExitCode(tt.progress); got != tt.want {
|
||||
t.Errorf("mirrorExitCode() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// proxy mirror used to exit 0 after printing "N failed", so a CI step that
|
||||
// prewarms the cache passed even when every download failed (#500). An
|
||||
// ecosystem the resolver rejects fails without touching the network.
|
||||
func TestMirrorExitsNonZeroWhenDownloadsFail(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorFailureProcess$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"PROXY_TEST_MIRROR_FAILURE=1",
|
||||
"PROXY_TEST_MIRROR_DIR="+dir,
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) {
|
||||
t.Fatalf("mirror with a failed download exited with %v, want status 1\n%s", err, output)
|
||||
}
|
||||
if code := exitErr.ExitCode(); code != 1 {
|
||||
t.Fatalf("exit status = %d, want 1\n%s", code, output)
|
||||
}
|
||||
if !strings.Contains(string(output), "1 failed") {
|
||||
t.Errorf("output does not report the failure:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(string(output), "bogus/foo@1.0.0") {
|
||||
t.Errorf("output does not list the failed package:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorFailureProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_MIRROR_FAILURE") != "1" {
|
||||
return
|
||||
}
|
||||
dir := os.Getenv("PROXY_TEST_MIRROR_DIR")
|
||||
os.Args = []string{
|
||||
"proxy", "mirror",
|
||||
"--storage-url", "file://" + filepath.Join(dir, "store"),
|
||||
"--database-path", filepath.Join(dir, "cache.db"),
|
||||
"pkg:bogus/foo@1.0.0",
|
||||
}
|
||||
main()
|
||||
}
|
||||
|
|
@ -14,58 +14,12 @@ base_url: "http://localhost:8080"
|
|||
# Set to "0" to disable the timeout. Default: "30s".
|
||||
# http_timeout: "30s"
|
||||
|
||||
# Response write deadline, including download and scan time before sending data.
|
||||
# Set to "0" to disable. Default: "5m".
|
||||
# write_timeout: "5m"
|
||||
|
||||
# Timeout for each upstream artifact download while serving requests; excludes
|
||||
# scanning. Set to "0" to disable. Default: "5m". Does not limit proxy mirror.
|
||||
# artifact_timeout: "5m"
|
||||
|
||||
# Register DELETE /api/cache and show Delete buttons on the UI's version page,
|
||||
# for removing cached artifacts so they are fetched again. Has no
|
||||
# authentication: restrict /api/cache and /ui at the ingress first.
|
||||
# cache_delete: false
|
||||
|
||||
# Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused
|
||||
# and not cached. Default: unlimited.
|
||||
# artifact_max_size: ""
|
||||
|
||||
# Memory for rewritten npm and Composer metadata, so each upstream document is
|
||||
# rewritten and gzip-compressed once rather than on every request. Set to "0"
|
||||
# to rewrite on every request. Default: "256MB".
|
||||
# metadata_rewrite_cache_size: "256MB"
|
||||
|
||||
# Public URL where the web UI is reached. Defaults to base_url when unset.
|
||||
# Set this separately when the UI is served on a different hostname than the
|
||||
# package endpoints — for example, the UI on a public domain behind auth while
|
||||
# build machines hit a Docker network alias for the package endpoints.
|
||||
# ui_base_url: "https://proxy.example.com/ui"
|
||||
|
||||
# Attribute requests to the leftmost X-Forwarded-For entry rather than the TCP
|
||||
# peer address, in the structured log, the access log and the request-source
|
||||
# table on /ui/analytics.
|
||||
#
|
||||
# Enable this only when the proxy sits behind a load balancer or ingress that
|
||||
# sets the header. Any client can send it: behind one it is the only way to see
|
||||
# past the hop, in front of one it lets a caller forge its own address.
|
||||
# trust_forwarded_for: false
|
||||
|
||||
# Show the request-source table on /ui/analytics: caller addresses, the tool
|
||||
# each ran and how much each pulled.
|
||||
#
|
||||
# Off by default. The proxy has no authentication of its own, so leave this off
|
||||
# unless /ui is gated by a reverse proxy; anyone who can reach the page can
|
||||
# otherwise read the addresses of your build fleet.
|
||||
# ui_request_sources: false
|
||||
|
||||
# List the latest failures behind each failure count on /ui/analytics (upstream,
|
||||
# storage, integrity, health probe and scan errors): the package, its upstream
|
||||
# URL or storage path, and the error. URLs lose credentials and query strings.
|
||||
#
|
||||
# Off by default, for the same reason as ui_request_sources.
|
||||
# ui_failure_details: false
|
||||
|
||||
# Artifact storage configuration
|
||||
storage:
|
||||
# Storage backend URL
|
||||
|
|
@ -98,32 +52,6 @@ storage:
|
|||
# Empty or "0" means unlimited
|
||||
max_size: ""
|
||||
|
||||
# Evict artifacts that have not been served for longer than a duration
|
||||
# ("30d", "72h"), independent of max_size. Resolution order: package rule,
|
||||
# then ecosystem rule, then default. "0" or empty means never evict by age.
|
||||
# The default only applies to ecosystems that support retention; naming an
|
||||
# ecosystem or package that does not is a configuration error. See
|
||||
# docs/configuration.md for the supported ecosystems.
|
||||
# Generic downloads require an explicit ecosystem or package rule;
|
||||
# they never inherit default, since an upstream release may disappear.
|
||||
retention:
|
||||
default: ""
|
||||
# ecosystems:
|
||||
# npm: "30d"
|
||||
# generic: "30d"
|
||||
# packages:
|
||||
# "pkg:npm/lodash": "0"
|
||||
# "pkg:generic/owner%2Frepo": "90d"
|
||||
# How often to look for expired artifacts; at least "1m".
|
||||
sweep_interval: "10m"
|
||||
|
||||
# Store fetched artifacts. Set to false to stream every download from
|
||||
# upstream without storing it; metadata filtering, cooldown and the
|
||||
# denylist still apply. Useful when another cache sits in front of the
|
||||
# proxy. false is incompatible with direct_serve, scanning, mirror_api,
|
||||
# mirror.schedules and mirror.refresh.
|
||||
cache_artifacts: true
|
||||
|
||||
# Redirect cached artifact downloads to presigned storage URLs (HTTP 302)
|
||||
# instead of streaming through the proxy. Only effective for S3, GCS, and Azure.
|
||||
# Leave disabled if clients reach the proxy through an authenticating gateway,
|
||||
|
|
@ -153,10 +81,6 @@ database:
|
|||
# Example: "postgres://user:password@localhost:5432/proxy?sslmode=disable"
|
||||
url: ""
|
||||
|
||||
# How often cache hit counts and last-access times are written, batched in
|
||||
# one transaction. "0" writes each hit as it happens.
|
||||
hit_flush_interval: "1s"
|
||||
|
||||
# Logging configuration
|
||||
log:
|
||||
# Minimum log level: "debug", "info", "warn", "error"
|
||||
|
|
@ -184,15 +108,6 @@ upstream:
|
|||
# gate on publish age, e.g. Yarn's npmMinimalAgeGate. Default: false.
|
||||
# npm_full_metadata: true
|
||||
|
||||
# npm packages served only from a private registry, by name pattern.
|
||||
# Matching packages are never looked up on the npm registry above.
|
||||
# npm_routes:
|
||||
# "@example/*": "https://npm.example.com/registry"
|
||||
|
||||
# Follow the npm registry's change feed, so a new release is fetched on the
|
||||
# next request instead of after metadata_ttl. Requires cache_metadata.
|
||||
# npm_change_feed: true
|
||||
|
||||
# Cargo sparse index URL
|
||||
cargo: "https://index.crates.io"
|
||||
|
||||
|
|
@ -223,12 +138,6 @@ upstream:
|
|||
# Maven repository URL (used by /maven endpoint)
|
||||
maven: "https://repo1.maven.org/maven2"
|
||||
|
||||
# Additional Maven repositories, served at /maven/{name}/. A name must not
|
||||
# be the start of a group ID (such as "org" or "tools"), since it would hide
|
||||
# those groups in the default repository.
|
||||
# maven_repositories:
|
||||
# grails: "https://repo.grails.org/grails/restricted"
|
||||
|
||||
# Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts)
|
||||
gradle_plugin_portal: "https://plugins.gradle.org/m2"
|
||||
|
||||
|
|
@ -244,23 +153,9 @@ upstream:
|
|||
# Packagist repository URL
|
||||
composer_repository: "https://repo.packagist.org"
|
||||
|
||||
# Composer packages served only from a private repository, by name pattern.
|
||||
# Matching packages are never looked up on composer_repository.
|
||||
# composer_routes:
|
||||
# "example/*": "https://composer.example.com/packages"
|
||||
|
||||
# Follow the change feed of the Packagist at composer, so a new release is
|
||||
# fetched on the next request instead of after metadata_ttl. Requires
|
||||
# cache_metadata.
|
||||
# composer_change_feed: true
|
||||
|
||||
# Conan registry URL
|
||||
conan: "https://center.conan.io"
|
||||
|
||||
# Chef Supermarket URL (used by /chef/). Point it at a private
|
||||
# Supermarket to proxy that instead.
|
||||
chef: "https://supermarket.chef.io"
|
||||
|
||||
# Conda channel base URL
|
||||
conda: "https://conda.anaconda.org"
|
||||
|
||||
|
|
@ -302,12 +197,6 @@ upstream:
|
|||
# oci:
|
||||
# ghcr: "https://ghcr.io"
|
||||
|
||||
# Registries a named OCI upstream mirrors, e.g. when ghcr above points at
|
||||
# an Artifactory remote of ghcr.io. containerd mirror requests whose ns
|
||||
# parameter names one of these hosts are served by that upstream.
|
||||
# oci_mirrors:
|
||||
# ghcr: ["ghcr.io"]
|
||||
|
||||
# Named Alpine APK repositories (used by /apk/{name}/).
|
||||
# Defaults to {"alpine": "https://dl-cdn.alpinelinux.org/alpine"} when empty;
|
||||
# configuring any entry replaces that default.
|
||||
|
|
@ -315,23 +204,6 @@ upstream:
|
|||
# alpine: "https://dl-cdn.alpinelinux.org/alpine"
|
||||
# private: "https://apk.example.com"
|
||||
|
||||
# Named Arch Linux pacman mirrors (used by /arch/{name}/).
|
||||
# Defaults to {"archlinux": "https://geo.mirror.pkgbuild.com"} when empty;
|
||||
# configuring any entry replaces that default. Arch Linux ARM needs a
|
||||
# regional HTTPS mirror of your choice, not the mirror.archlinuxarm.org
|
||||
# redirector.
|
||||
# arch:
|
||||
# archlinux: "https://geo.mirror.pkgbuild.com"
|
||||
# archlinuxarm: "https://<region>.mirror.archlinuxarm.org"
|
||||
|
||||
# Named FreeBSD pkg mirrors (used by /freebsd/{name}/), for ports and
|
||||
# pkgbase repositories alike.
|
||||
# Defaults to {"freebsd": "https://pkg.freebsd.org"} when empty;
|
||||
# configuring any entry replaces that default.
|
||||
# freebsd:
|
||||
# freebsd: "https://pkg.freebsd.org"
|
||||
# poudriere: "https://packages.example.com"
|
||||
|
||||
# Named generic HTTP upstreams (used by /generic/{name}/). The remaining
|
||||
# request path and query are appended to the upstream URL. GitHub release
|
||||
# assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached
|
||||
|
|
@ -339,36 +211,6 @@ upstream:
|
|||
# generic:
|
||||
# github: "https://github.com"
|
||||
# github-api: "https://api.github.com"
|
||||
# apache: "https://archive.apache.org"
|
||||
|
||||
# Paths a generic upstream may serve, as path.Match patterns for the path
|
||||
# after /generic/{name}/; a pattern ending in /** also matches everything
|
||||
# nested below its prefix. Other paths answer 404. Required for an upstream
|
||||
# that upstream.auth has credentials for, such as a GitHub token for
|
||||
# api.github.com; use a fine-grained token with read-only access to public
|
||||
# repositories.
|
||||
# generic_paths:
|
||||
# github-api:
|
||||
# - "repos/*/*/releases"
|
||||
# - "repos/*/*/releases/**"
|
||||
# - "repos/*/*/tags"
|
||||
|
||||
# Version-pinned paths of generic upstreams, cached immutably like GitHub
|
||||
# release assets. Each pattern must match the whole path after
|
||||
# /generic/{name}/ and have `version` and `file` named groups; an optional
|
||||
# `name` group names the package.
|
||||
# generic_artifacts:
|
||||
# apache:
|
||||
# - 'dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)'
|
||||
|
||||
# Named Git servers (used by /git/{name}/{repo}), served read-only from
|
||||
# cached bare repositories. repos limits which repositories may be served
|
||||
# and is required when upstream.auth has credentials for the server.
|
||||
# git:
|
||||
# github: "https://github.com"
|
||||
# wikimedia:
|
||||
# url: "https://gerrit.wikimedia.org/r"
|
||||
# repos: ["mediawiki/**"]
|
||||
|
||||
# Authentication for upstream registries
|
||||
# Keys are absolute URL scopes. Scheme, host, effective port, and path
|
||||
|
|
@ -410,20 +252,11 @@ upstream:
|
|||
# "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com":
|
||||
# type: ecr
|
||||
|
||||
# Git repository cache, used when upstream.git is set
|
||||
# git:
|
||||
# cache_dir: ./cache/git # local disk, used by this process only
|
||||
# max_size: "" # evict least recently used repositories past this
|
||||
# max_repo_size: 5GB # refuse a repository that grows past this
|
||||
# refresh_wait: 5s # then serve the cached copy if the upstream check is slow
|
||||
# stale_max_age: 24h # never serve a copy not checked for this long
|
||||
|
||||
# Gradle HttpBuildCache configuration
|
||||
gradle:
|
||||
build_cache:
|
||||
# PUT uploads are rejected by default; set to false on the instance
|
||||
# your CI jobs upload to
|
||||
read_only: true
|
||||
# Set to true to disable PUT uploads (read-only cache mode)
|
||||
read_only: false
|
||||
|
||||
# Maximum accepted Gradle cache upload body size
|
||||
# Required and must be > 0
|
||||
|
|
@ -439,45 +272,6 @@ gradle:
|
|||
# How often eviction runs when max_age or max_size is set
|
||||
sweep_interval: "10m"
|
||||
|
||||
# Pushing package archives to /api/hosted (disabled by default).
|
||||
# A pushed version replaces the upstream one for every client, so a token is
|
||||
# required to push and delete.
|
||||
# hosted:
|
||||
# enabled: true
|
||||
# token: "${PROXY_HOSTED_TOKEN}"
|
||||
# max_upload_size: "100MB"
|
||||
# # Namespaces served from pushed packages only; the upstream is never asked
|
||||
# # about them, so nobody can mix in versions by publishing the same name.
|
||||
# npm_scopes: ["@acme"]
|
||||
# composer_vendors: ["acme"]
|
||||
|
||||
# Project inventory (disabled by default). Repositories upload an SBOM to
|
||||
# PUT /api/projects/{project}/sbom, and the UI shows which project uses
|
||||
# which package versions. Project names are visible on the unauthenticated UI.
|
||||
# projects:
|
||||
# enabled: true
|
||||
# token: "${PROXY_PROJECTS_TOKEN}"
|
||||
# # Drop projects that have not uploaded for this long ("" keeps them)
|
||||
# max_age: "90d"
|
||||
|
||||
# Re-mirror container image tags on a schedule, so a moved tag is picked up
|
||||
# even when nobody pulls it. An unchanged tag costs one HEAD request.
|
||||
# mirror:
|
||||
# schedules:
|
||||
# - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"]
|
||||
# interval: "6h"
|
||||
# platforms: ["linux/amd64"]
|
||||
#
|
||||
# Mirror new releases of the most used packages in the background, so the
|
||||
# first install after a release doesn't wait on the upstream. Cooldown and
|
||||
# the denylist apply.
|
||||
# refresh:
|
||||
# interval: "24h"
|
||||
# top: 50 # packages per ecosystem, by recent downloads
|
||||
# since: "14d"
|
||||
# max_versions: 200 # per run; default no cap
|
||||
# max_bytes: "2GB" # per run; default no cap
|
||||
|
||||
# Health endpoint configuration.
|
||||
health:
|
||||
# Minimum time between storage backend probes.
|
||||
|
|
@ -492,13 +286,6 @@ health:
|
|||
# to spot malicious releases before they're pulled into projects.
|
||||
# Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled)
|
||||
cooldown:
|
||||
# Independent policies selected through /cooldown/{name}/{registry}/.
|
||||
# profiles:
|
||||
# 3d:
|
||||
# default: "3d"
|
||||
# 7d:
|
||||
# default: "7d"
|
||||
|
||||
# Global default cooldown for all ecosystems
|
||||
# default: "3d"
|
||||
|
||||
|
|
@ -513,10 +300,6 @@ cooldown:
|
|||
# "pkg:npm/lodash": "0"
|
||||
# "pkg:npm/@babel/core": "14d"
|
||||
|
||||
# Per-package glob overrides, after exact packages and before ecosystems.
|
||||
# package_patterns:
|
||||
# "pkg:npm/@example/*": "0"
|
||||
|
||||
# Exact versions to deny, independently of cooldown and scanning.
|
||||
# Metadata filtering: npm, PyPI and Cargo. Shared artifact downloads, including
|
||||
# cache hits, are blocked with 403; signed APT metadata is left unchanged.
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue