Watch
1
0
Fork
You've already forked pkg-proxy
1
mirror of https://github.com/git-pkgs/proxy.git synced 2026-11-09 15:49:16 -05:00

Compare commits

..
561 changed files with 5995 additions and 124097 deletions

View file

@ -1,17 +0,0 @@
# Version and commit come in as build args, so the build doesn't need .git.
.git
.github
# Local build output and runtime data
/proxy
dist/
build/
cache/
*.db
*.tar.gz
# Local config and secrets
.env
.env.local
proxy.cfg
config.yaml

View file

@ -13,17 +13,18 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-latest, macos-latest, windows-latest] os: [ubuntu-latest, macos-latest, windows-latest]
go-version: ['1.25']
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with: with:
go-version-file: go.mod go-version: ${{ matrix.go-version }}
- name: Build - name: Build
run: go build -v ./... run: go build -v ./...
@ -31,92 +32,17 @@ jobs:
- name: Test - name: Test
run: go test -v -race ./... run: go test -v -race ./...
postgres:
name: test (postgres)
runs-on: ubuntu-latest
services:
postgres:
image: postgres:17@sha256:b8faeff7712663c2773a190181e1246e37e364b887611c04658c28b44e2408fe
env:
POSTGRES_PASSWORD: test
POSTGRES_DB: proxy
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres -d proxy"
--health-interval 2s
--health-timeout 5s
--health-retries 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
# The packages with tests that need PROXY_DATABASE_URL. -p 1 because
# their tests drop and recreate tables in the one shared database.
- name: Test
env:
PROXY_DATABASE_URL: postgres://postgres:test@localhost:5432/proxy?sslmode=disable
run: go test -v -race -p 1 ./internal/database/... ./internal/handler/... ./internal/server/...
lint: lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with: with:
go-version-file: go.mod go-version: '1.25'
- name: Install golangci-lint
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0
- name: golangci-lint - name: golangci-lint
run: golangci-lint run ./... run: go tool golangci-lint run ./...
helm:
name: Helm chart
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
with:
version: v3.18.6
- name: Lint chart
run: helm lint deploy/charts/proxy
- name: Render chart variants
run: |
set -euo pipefail
helm template proxy deploy/charts/proxy >/dev/null
helm template proxy deploy/charts/proxy \
--set persistence.enabled=false \
--set config.existingConfigMap=proxy-config \
--set ingress.enabled=true \
--set 'ingress.hosts[0].host=proxy.example.com' \
--set 'ingress.hosts[0].paths[0].path=/' \
--set 'ingress.hosts[0].paths[0].pathType=Prefix' \
>/dev/null
helm template proxy deploy/charts/proxy \
--set 'extraVolumes[0].name=corp-ca' \
--set 'extraVolumes[0].configMap.name=corp-ca' \
--set 'extraVolumeMounts[0].name=corp-ca' \
--set 'extraVolumeMounts[0].mountPath=/etc/proxy/ca' \
--set 'extraVolumeMounts[0].readOnly=true' \
--set 'extraInitContainers[0].name=wait-for-db' \
--set 'extraInitContainers[0].image=busybox:1.37' \
--set 'extraInitContainers[0].command={sh,-c,exit 0}' \
>/dev/null

View file

@ -1,137 +0,0 @@
name: Monthly release
# Cuts a release on the 1st of each month when main is green and something
# has merged since the last release: a patch release, or a new minor version
# if a pull request labelled "breaking" was merged (a new major version from
# 1.0 on). A breaking release is created as a draft, and its images and chart
# are published only when someone publishes the draft, after writing the
# upgrade notes. Scheduled runs only report what they would do until the
# repository variable MONTHLY_RELEASE is "true".
on:
schedule:
- cron: "0 6 1 * *"
workflow_dispatch:
inputs:
dry_run:
description: Only report what would be released
type: boolean
default: true
permissions: {}
concurrency:
group: monthly-release
jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: write
actions: write
checks: read
pull-requests: read
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: main
fetch-depth: 0
persist-credentials: false
- name: Decide whether to release
id: plan
env:
EVENT: ${{ github.event_name }}
INPUT_DRY_RUN: ${{ inputs.dry_run }}
ENABLED: ${{ vars.MONTHLY_RELEASE }}
run: |
set -euo pipefail
if [ "$EVENT" = workflow_dispatch ]; then
dry_run=$INPUT_DRY_RUN
elif [ "$ENABLED" = true ]; then
dry_run=false
else
dry_run=true
fi
echo "dry_run=$dry_run" >> "$GITHUB_OUTPUT"
sha=$(git rev-parse HEAD)
last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*')
# A breaking release waits as a draft for its upgrade notes. Building
# on its tag would publish those changes without them. Older tags
# were released by hand and may have no release at all.
draft=$(gh release view "$last" -R "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo false)
if [ "$draft" = true ]; then
echo "::warning::$last is still a draft, publish it first"
exit 0
fi
merged=$(git rev-list --count "$last..$sha")
if [ "$merged" -eq 0 ]; then
echo "Nothing merged since $last"
exit 0
fi
# Every check on main's HEAD must have finished green. This run
# shows up as a check on the same commit, so it is left out.
bad=$(gh api --paginate "repos/$REPO/commits/$sha/check-runs?per_page=100" \
--jq ".check_runs[]
| select(.html_url | contains(\"/actions/runs/$GITHUB_RUN_ID/\") | not)
| select(.status != \"completed\"
or (.conclusion | IN(\"success\", \"skipped\", \"neutral\") | not))
| .name")
if [ -n "$bad" ]; then
echo "::warning::main ($sha) is not green, skipping: $(echo "$bad" | sort -u | paste -sd, -)"
exit 0
fi
since=$(git log -1 --format=%cI "$last")
breaking=$(gh pr list -R "$REPO" --state merged --base main --limit 1000 \
--search "merged:>$since label:breaking" --json number --jq '.[].number')
IFS=. read -r major minor patch <<<"${last#v}"
if [ -z "$breaking" ]; then
tag="v$major.$minor.$((patch + 1))"
else
echo "Breaking changes merged since $last: #$(echo "$breaking" | sort -n | paste -sd, - | sed 's/,/, #/g')"
if [ "$major" -eq 0 ]; then
tag="v0.$((minor + 1)).0"
else
tag="v$((major + 1)).0.0"
fi
fi
echo "$merged commits since $last, next release $tag at $sha"
echo "tag=$tag" >> "$GITHUB_OUTPUT"
echo "sha=$sha" >> "$GITHUB_OUTPUT"
echo "draft=$([ -n "$breaking" ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
# A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the
# release workflows are dispatched on the new tag instead.
- name: Tag and start the release workflows
if: steps.plan.outputs.tag != ''
env:
TAG: ${{ steps.plan.outputs.tag }}
SHA: ${{ steps.plan.outputs.sha }}
DRY_RUN: ${{ steps.plan.outputs.dry_run }}
DRAFT: ${{ steps.plan.outputs.draft }}
run: |
set -euo pipefail
if [ "$DRAFT" = true ]; then
what="start release.yml for a draft release"
else
what="start release.yml and publish.yml"
fi
if [ "$DRY_RUN" = true ]; then
echo "Dry run: would tag $TAG at $SHA and $what"
exit 0
fi
gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null
gh workflow run release.yml -R "$REPO" --ref "$TAG" -f draft="$DRAFT"
if [ "$DRAFT" = true ]; then
echo "::notice::$TAG is a draft release: add the upgrade notes and publish it to push the images and chart"
else
gh workflow run publish.yml -R "$REPO" --ref "$TAG"
fi
echo "Tagged $TAG at $SHA"

View file

@ -5,10 +5,6 @@ on:
push: push:
tags: tags:
- "v*" - "v*"
# A draft release made by monthly-release.yml is published by hand. Releases
# published with GITHUB_TOKEN don't start this.
release:
types: [published]
permissions: {} permissions: {}
@ -22,23 +18,14 @@ jobs:
permissions: permissions:
packages: write packages: write
contents: read contents: read
id-token: write
steps: steps:
- name: Check out the repo - name: Check out the repo
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with: with:
persist-credentials: false persist-credentials: false
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Log in to the Container registry - name: Log in to the Container registry
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
@ -46,123 +33,14 @@ jobs:
- name: Extract metadata (tags, labels) for Docker - name: Extract metadata (tags, labels) for Docker
id: meta id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9
with: with:
images: ghcr.io/${{ github.repository }} images: ghcr.io/${{ github.repository }}
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Build and push Docker image - name: Build and push Docker image
id: build uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc
with: with:
context: . context: .
platforms: linux/amd64,linux/arm64
push: true push: true
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
COMMIT=${{ github.sha }}
provenance: mode=max
sbom: true
- name: Sign image by digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
cosign sign --yes "${IMAGE}@${DIGEST}"
- name: Verify BuildKit attestations and extract SPDX predicates
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
reference="${IMAGE}@${DIGEST}"
docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json
docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json
for platform in linux/amd64 linux/arm64; do
jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \
provenance.json >/dev/null
jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json"
done
- name: Attest platform SBOMs by digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
reference="${IMAGE}@${DIGEST}"
for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do
cosign attest --yes --type spdxjson --predicate "$predicate" "$reference"
done
publish_chart:
name: Push Helm chart to GHCR
if: github.ref_type == 'tag'
needs: push_to_registry
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Check out the repo
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
ref: ${{ github.sha }}
- uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
with:
version: v3.18.6
- name: Validate and normalize release version
id: version
env:
TAG: ${{ github.ref_name }}
run: |
set -euo pipefail
semver='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?$'
[[ "$TAG" =~ $semver ]] || {
echo "Tag must be strict SemVer of the form vMAJOR.MINOR.PATCH[-PRERELEASE]: $TAG" >&2
exit 1
}
version="${TAG#v}"
[[ "$version" != "0.0.0" ]] || {
echo "0.0.0 is a development placeholder and must not be published" >&2
exit 1
}
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Log in to GHCR
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: printf '%s' "$GH_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
- name: Lint and package chart
env:
VERSION: ${{ steps.version.outputs.version }}
run: |
set -euo pipefail
helm lint deploy/charts/proxy
mkdir -p build
helm package \
--destination build \
--version "$VERSION" \
--app-version "$VERSION" \
deploy/charts/proxy
metadata="$(helm show chart "build/proxy-${VERSION}.tgz")"
[[ "$(awk '$1 == "version:" {print $2}' <<<"$metadata")" == "$VERSION" ]]
[[ "$(awk '$1 == "appVersion:" {gsub(/\"/, "", $2); print $2}' <<<"$metadata")" == "$VERSION" ]]
- name: Push chart
env:
VERSION: ${{ steps.version.outputs.version }}
run: helm push "build/proxy-${VERSION}.tgz" oci://ghcr.io/git-pkgs/charts

View file

@ -4,13 +4,6 @@ on:
push: push:
tags: tags:
- "v*" - "v*"
# Started by monthly-release.yml on the tag it creates.
workflow_dispatch:
inputs:
draft:
description: Create the release as a draft
type: boolean
default: false
permissions: permissions:
contents: write contents: write
@ -18,11 +11,10 @@ permissions:
jobs: jobs:
release: release:
if: github.ref_type == 'tag'
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with: with:
fetch-depth: 0 fetch-depth: 0
persist-credentials: false persist-credentials: false
@ -30,14 +22,14 @@ jobs:
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with: with:
go-version-file: go.mod go-version-file: go.mod
cache: false cache: false
- uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 - uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2
with: with:
version: "~> v2" version: "~> v2"
args: release --clean${{ inputs.draft && ' --draft' || '' }} args: release --clean
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -12,17 +12,17 @@ jobs:
swagger: swagger:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with: with:
go-version-file: go.mod go-version: '1.25'
- name: Install swag - name: Install swag
run: go install github.com/swaggo/swag/cmd/swag@v1.16.6 run: go install github.com/swaggo/swag/cmd/swag@latest
- name: Generate swagger - name: Generate swagger
run: go generate ./internal/server run: go generate ./internal/server

View file

@ -5,22 +5,14 @@ on:
branches: branches:
- main - main
paths: paths:
- '.github/**' - '.github/workflows/**'
- '**/action.yml'
- '**/action.yaml'
- 'zizmor.yml'
- 'zizmor.yaml'
pull_request: pull_request:
branches:
- main
paths: paths:
- '.github/**' - '.github/workflows/**'
- '**/action.yml'
- '**/action.yaml'
- 'zizmor.yml'
- 'zizmor.yaml'
workflow_dispatch: workflow_dispatch:
permissions: {}
jobs: jobs:
zizmor: zizmor:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -29,9 +21,9 @@ jobs:
security-events: write security-events: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with: with:
persist-credentials: false persist-credentials: false
- name: Run zizmor - name: Run zizmor
uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7

4
.gitignore vendored
View file

@ -4,7 +4,7 @@
*.dll *.dll
*.so *.so
*.dylib *.dylib
/proxy proxy
# Test binary, built with `go test -c` # Test binary, built with `go test -c`
*.test *.test
@ -43,4 +43,4 @@ cache/*
# Debug files # Debug files
__debug_bin __debug_bin
debug debug

View file

@ -26,7 +26,3 @@ linters:
- goconst - goconst
- dupl - dupl
- mnd - mnd
formatters:
enable:
- gofmt

View file

@ -15,7 +15,6 @@ builds:
- linux - linux
- darwin - darwin
- windows - windows
- freebsd
goarch: goarch:
- amd64 - amd64
- arm64 - arm64

View file

@ -39,7 +39,7 @@ proxy/
│ │ └── queries.go # CRUD operations │ │ └── queries.go # CRUD operations
│ ├── storage/ # Artifact file storage │ ├── storage/ # Artifact file storage
│ │ ├── storage.go # Storage interface │ │ ├── storage.go # Storage interface
│ │ └── blob.go # gocloud.dev/blob backends (file, S3, Azure) │ │ └── filesystem.go # Local filesystem impl
│ ├── upstream/ # Upstream registry clients │ ├── upstream/ # Upstream registry clients
│ │ ├── fetcher.go # HTTP artifact fetching │ │ ├── fetcher.go # HTTP artifact fetching
│ │ └── resolver.go # Download URL resolution │ │ └── resolver.go # Download URL resolution
@ -72,7 +72,7 @@ Key types:
### `internal/storage` ### `internal/storage`
Artifact file storage abstraction backed by `gocloud.dev/blob`. Supports local filesystem (`file://`), S3 (`s3://`), and Azure (`azblob://`) URLs. Artifact file storage abstraction. Currently implements local filesystem storage. Designed to allow future backends (S3, GCS).
Interface: Interface:
```go ```go
@ -160,7 +160,6 @@ mux.Handle("/newregistry/", http.StripPrefix("/newregistry", newHandler.Routes()
- Keep functions short and focused - Keep functions short and focused
- Write tests for new functionality - Write tests for new functionality
- Document exported types and functions - Document exported types and functions
- A new Prometheus metric needs a tile on `/ui/analytics` and an entry in `metricSurface` (`internal/server/analytics_coverage_test.go`). The page is meant to be a complete view of `/metrics`, so `TestEveryMetricIsSurfaced` fails until both exist.
## Testing ## Testing
@ -212,24 +211,6 @@ Add PyPI registry support
- Add tests for wheel and sdist downloads - Add tests for wheel and sdist downloads
``` ```
## Reviewing and Merging
Maintainers are the members of the proxy team in the git-pkgs org. Any maintainer can review and merge. A pull request needs green CI and an approval from a maintainer other than its author. A "request changes" review from any maintainer blocks it until that reviewer approves or withdraws the review.
Using AI to write or review is fine, but whoever approves a pull request should understand it and stand behind it. If it touches an ecosystem you don't use, you can ask someone who does to look before you approve.
Open an issue before a bigger change: anything that changes behaviour users rely on, new configuration that would be hard to remove, or a change that fans out into other git-pkgs modules. If no maintainer objects within a few days, open the pull request; it's reviewed like any other. Objections are worked out in the issue, with @andrew breaking ties for now. Where possible, deprecate before removing: an old configuration key keeps working, with a warning, in at least one release before the one that removes it.
## Releases
A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on.
A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. Until then the monthly run releases nothing.
Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default.
To release by hand, push a `v*` tag.
## Questions? ## Questions?
Open an issue on GitHub if you have questions or need help getting started. Open an issue on GitHub if you have questions or need help getting started.

View file

@ -1,4 +1,4 @@
FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS builder FROM golang:1.26.4-alpine AS builder
WORKDIR /src WORKDIR /src
@ -12,17 +12,12 @@ RUN go mod download
# Copy source code # Copy source code
COPY . . COPY . .
# Build the binary for the target platform # Build the binary
ARG TARGETARCH RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
ARG VERSION=dev
ARG COMMIT=unknown
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
-ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \
-o /proxy ./cmd/proxy
FROM alpine:3.24.3 FROM alpine:3.24.1
RUN apk add --no-cache ca-certificates git RUN apk add --no-cache ca-certificates
COPY --from=builder /proxy /usr/local/bin/proxy COPY --from=builder /proxy /usr/local/bin/proxy
@ -35,9 +30,5 @@ WORKDIR /data
EXPOSE 8080 EXPOSE 8080
# /health checks the database and storage and can take up to ten seconds.
HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \
CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1
ENTRYPOINT ["proxy"] ENTRYPOINT ["proxy"]
CMD ["serve"] CMD ["serve"]

960
README.md

File diff suppressed because it is too large Load diff

View file

@ -1,278 +0,0 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"text/tabwriter"
"time"
"github.com/git-pkgs/proxy/internal/handler"
)
const (
defaultProxyURL = "http://localhost:8080"
hostedTokenEnv = "PROXY_HOSTED_TOKEN"
hostedClientTimeout = 10 * time.Minute
pushArgCount = 4 // ecosystem, name, version, file
listArgCount = 2 // ecosystem, name
deleteArgCount = 3 // ecosystem, name, version
deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files
tablePadding = 2
)
// hostedArtifact mirrors the JSON the /api/hosted endpoints return.
type hostedArtifact struct {
Version string `json:"version"`
Filename string `json:"filename"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
PushedAt time.Time `json:"pushed_at"`
}
type hostedPushResponse struct {
Ecosystem string `json:"ecosystem"`
Name string `json:"name"`
hostedArtifact
}
type hostedListResponse struct {
Ecosystem string `json:"ecosystem"`
Name string `json:"name"`
Artifacts []hostedArtifact `json:"artifacts"`
}
type hostedCommand struct {
flags *flag.FlagSet
proxyURL *string
usage string
}
func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand {
fs := flag.NewFlagSet(name, flag.ContinueOnError)
fs.SetOutput(io.Discard)
proxyURL := getenv("PROXY_URL")
if proxyURL == "" {
proxyURL = defaultProxyURL
}
cmd := hostedCommand{flags: fs, usage: usage}
cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)")
fs.Usage = func() {
fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage)
fs.SetOutput(os.Stderr)
fs.PrintDefaults()
}
return cmd
}
// parse parses flags and checks for between minArgs and maxArgs positional
// arguments.
func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) {
if err := c.flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
c.flags.Usage()
}
return nil, fmt.Errorf("%w\nusage: %s", err, c.usage)
}
if n := c.flags.NArg(); n < minArgs || n > maxArgs {
return nil, fmt.Errorf("usage: %s", c.usage)
}
return c.flags.Args(), nil
}
func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string {
segments := []string{url.PathEscape(ecosystem)}
for segment := range strings.SplitSeq(name, "/") {
segments = append(segments, url.PathEscape(segment))
}
for _, v := range version {
segments = append(segments, url.PathEscape(v))
}
return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/")
}
func hostedToken(getenv func(string) string) (string, error) {
token := getenv(hostedTokenEnv)
if token == "" {
return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv)
}
return token, nil
}
func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
cmd := newHostedCommand("push", "proxy hosted push [-url URL] <ecosystem> <name> <version> <file>",
"Push a package archive to the proxy", getenv)
positional, err := cmd.parse(args, pushArgCount, pushArgCount)
if err != nil {
return err
}
token, err := hostedToken(getenv)
if err != nil {
return err
}
ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3]
file, err := os.Open(path)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
// The proxy checks the upload against this, so a truncated or corrupted
// transfer is refused rather than stored.
digest := sha256.New()
size, err := io.Copy(digest, file)
if err != nil {
return err
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
segments := []string{version}
if handler.HostedFileInPath(ecosystem) {
segments = append(segments, filepath.Base(path))
}
endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil))
req, err := http.NewRequest(http.MethodPut, endpoint, file)
if err != nil {
return err
}
req.ContentLength = size
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/octet-stream")
// The proxy refuses an existing file or a bad token before reading the
// body, so the file is only uploaded once it would be accepted.
req.Header.Set("Expect", "100-continue")
var pushed hostedPushResponse
if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil {
return err
}
_, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n",
pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256)
return err
}
func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
cmd := newHostedCommand("list", "proxy hosted list [-url URL] <ecosystem> <name>", "List pushed package archives", getenv)
positional, err := cmd.parse(args, listArgCount, listArgCount)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil)
if err != nil {
return err
}
var listed hostedListResponse
if err := doHosted(client, req, http.StatusOK, &listed); err != nil {
return err
}
tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0)
_, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED")
for _, a := range listed.Artifacts {
_, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339))
}
return tw.Flush()
}
func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] <ecosystem> <name> <version> [file]",
"Delete a pushed version, or one generic file of it", getenv)
positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount)
if err != nil {
return err
}
if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) {
return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage)
}
token, err := hostedToken(getenv)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+token)
if err := doHosted(client, req, http.StatusNoContent, nil); err != nil {
return err
}
deleted := positional[1] + "@" + positional[2]
if len(positional) > deleteArgCount {
deleted += " " + positional[deleteArgCount]
}
_, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted)
return err
}
// doHosted sends req and decodes a successful response into out. Any other
// status is returned as an error carrying the server's message.
func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error {
resp, err := client.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != wantStatus {
var apiErr struct {
Message string `json:"message"`
}
if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" {
return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status)
}
return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status)
}
if out == nil {
return nil
}
return json.NewDecoder(resp.Body).Decode(out)
}
const hostedUsage = "usage: proxy hosted <push|list|delete> [flags] ...\n\n" +
" push Push a package archive to the proxy\n" +
" list List pushed package archives\n" +
" delete Delete a pushed version, or one generic file of it"
type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error
var hostedSubcommands = map[string]hostedSubcommand{
"push": runPush,
"list": runList,
"delete": runDelete,
}
// runHosted runs the hosted subcommand named by args[0].
func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
if len(args) == 0 {
return errors.New(hostedUsage)
}
run, ok := hostedSubcommands[args[0]]
if !ok {
return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage)
}
return run(args[1:], getenv, stdout, client)
}
// runHostedCommand runs proxy hosted against the proxy's /api/hosted
// endpoints and exits non-zero on failure.
func runHostedCommand() {
client := &http.Client{Timeout: hostedClientTimeout}
if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

View file

@ -1,220 +0,0 @@
package main
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
type recordedRequest struct {
method, path, query, authorization, body, expect string
contentLength int64
}
func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) {
t.Helper()
recorded := &recordedRequest{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
*recorded = recordedRequest{
method: r.Method,
path: r.URL.EscapedPath(),
query: r.URL.RawQuery,
authorization: r.Header.Get("Authorization"),
body: string(body),
expect: r.Header.Get("Expect"),
contentLength: r.ContentLength,
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_, _ = io.WriteString(w, response)
}))
t.Cleanup(srv.Close)
return srv, recorded
}
func hostedEnv(vars map[string]string) func(string) string {
return func(key string) string { return vars[key] }
}
func writeArchive(t *testing.T, name, content string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatalf("writing archive: %v", err)
}
return path
}
func TestRunPush(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusCreated,
`{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`)
archive := writeArchive(t, "node-20.1.0.tgz", "tarball")
var stdout bytes.Buffer
err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
if err != nil {
t.Fatalf("runPush: %v", err)
}
if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" {
t.Errorf("request = %s %s", recorded.method, recorded.path)
}
// sha256 of "tarball", so the proxy can check the upload.
const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0"
if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 {
t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query)
}
if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) {
t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength)
}
if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") {
t.Errorf("output = %q, want filename and sha256", out)
}
}
func TestRunPushGenericNamesTheFile(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusCreated,
`{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`)
archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball")
err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
if err != nil {
t.Fatalf("runPush: %v", err)
}
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
t.Errorf("path = %q, want the file name appended", recorded.path)
}
}
func TestRunPushErrors(t *testing.T) {
srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`)
archive := writeArchive(t, "serde-1.0.200.crate", "crate")
err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client())
if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") {
t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err)
}
err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") {
t.Errorf("rejected push error = %v, want the server's message", err)
}
err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
if err == nil || !strings.Contains(err.Error(), "usage") {
t.Errorf("push with missing arguments error = %v, want usage", err)
}
}
func TestRunPushUsesProxyURLFromEnv(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`)
archive := writeArchive(t, "request-2.88.3.tgz", "tarball")
err := runPush([]string{"npm", "request", "2.88.3", archive},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client())
if err != nil {
t.Fatalf("runPush: %v", err)
}
if recorded.path != "/api/hosted/npm/request/2.88.3" {
t.Errorf("path = %q", recorded.path)
}
}
func TestRunList(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusOK,
`{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`)
var stdout bytes.Buffer
if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil {
t.Fatalf("runList: %v", err)
}
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" {
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
}
if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") {
t.Errorf("output = %q", out)
}
}
func TestRunDelete(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
if err != nil {
t.Fatalf("runDelete: %v", err)
}
if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" {
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
}
missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`)
err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client())
if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") {
t.Errorf("delete of a missing version error = %v", err)
}
}
func TestRunDeleteOneFile(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
var stdout bytes.Buffer
err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
if err != nil {
t.Fatalf("runDelete: %v", err)
}
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
t.Errorf("path = %q", recorded.path)
}
if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" {
t.Errorf("output = %q", out)
}
}
func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"},
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
if err == nil || !strings.Contains(err.Error(), "only generic files") {
t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err)
}
if recorded.method != "" {
t.Errorf("refused delete sent %s %s", recorded.method, recorded.path)
}
}
func TestRunHostedDispatchesSubcommands(t *testing.T) {
srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`)
if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil {
t.Fatalf("runHosted list: %v", err)
}
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" {
t.Errorf("request = %s %s", recorded.method, recorded.path)
}
for _, args := range [][]string{nil, {"bogus"}} {
err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client())
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted <push|list|delete>") {
t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err)
}
}
}
func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) {
err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient)
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") {
t.Errorf("push usage error = %v, want it to name proxy hosted push", err)
}
}

View file

@ -5,11 +5,6 @@
// @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing. // @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing.
// @BasePath / // @BasePath /
// //
// @securityDefinitions.apikey BearerAuth
// @in header
// @name Authorization
// @description "Bearer " followed by hosted.token, or projects.token for /api/projects
//
// The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // The proxy caches package artifacts from upstream registries (npm, cargo, etc.)
// providing faster, more reliable access for development teams. // providing faster, more reliable access for development teams.
// //
@ -22,7 +17,6 @@
// serve Start the proxy server (default if no command given) // serve Start the proxy server (default if no command given)
// stats Show cache statistics // stats Show cache statistics
// mirror Pre-populate cache from PURLs, SBOMs, or registries // mirror Pre-populate cache from PURLs, SBOMs, or registries
// hosted Push, list or delete packages on a running proxy (needs hosted.enabled)
// //
// Serve Flags: // Serve Flags:
// //
@ -46,8 +40,6 @@
// Log level: debug, info, warn, error (default "info") // Log level: debug, info, warn, error (default "info")
// -log-format string // -log-format string
// Log format: text, json (default "text") // Log format: text, json (default "text")
// -access-log string
// Path to the JSONL access log (disabled by default)
// //
// Stats Flags: // Stats Flags:
// //
@ -80,23 +72,14 @@
// PROXY_DATABASE_URL - PostgreSQL connection URL // PROXY_DATABASE_URL - PostgreSQL connection URL
// PROXY_LOG_LEVEL - Log level // PROXY_LOG_LEVEL - Log level
// PROXY_LOG_FORMAT - Log format // PROXY_LOG_FORMAT - Log format
// PROXY_ACCESS_LOG_PATH - JSONL access log path // PROXY_UPSTREAM_MAVEN - Maven repository upstream URL
// PROXY_UPSTREAM_* - Upstream URLs and network access controls // PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Set false to allow Gradle PUT uploads (default true) // PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size // PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction // PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval // PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s") // PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted
// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting
// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB")
// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only
// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only
// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects)
// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs
// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d")
// PROXY_URL - Proxy URL for proxy hosted
// //
// Example: // Example:
// //
@ -121,17 +104,16 @@ import (
"log/slog" "log/slog"
"os" "os"
"os/signal" "os/signal"
"runtime/debug"
"strings" "strings"
"syscall" "syscall"
"github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/config"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/handler"
"github.com/git-pkgs/proxy/internal/memlimit"
"github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/mirror"
"github.com/git-pkgs/proxy/internal/server" "github.com/git-pkgs/proxy/internal/server"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/registries/fetch"
) )
const defaultTopN = 10 const defaultTopN = 10
@ -144,15 +126,6 @@ var (
Commit = "unknown" Commit = "unknown"
) )
func init() {
if Version != "dev" {
return
}
if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" {
Version = bi.Main.Version
}
}
func main() { func main() {
if len(os.Args) > 1 { if len(os.Args) > 1 {
switch os.Args[1] { switch os.Args[1] {
@ -168,10 +141,6 @@ func main() {
os.Args = append(os.Args[:1], os.Args[2:]...) os.Args = append(os.Args[:1], os.Args[2:]...)
runMirror() runMirror()
return return
case "hosted":
os.Args = append(os.Args[:1], os.Args[2:]...)
runHostedCommand()
return
case "-version", "--version": case "-version", "--version":
fmt.Printf("proxy %s (%s)\n", Version, Commit) fmt.Printf("proxy %s (%s)\n", Version, Commit)
os.Exit(0) os.Exit(0)
@ -194,7 +163,6 @@ Commands:
serve Start the proxy server (default) serve Start the proxy server (default)
stats Show cache statistics stats Show cache statistics
mirror Pre-populate cache from PURLs, SBOMs, or registries mirror Pre-populate cache from PURLs, SBOMs, or registries
hosted Push, list or delete packages on a running proxy
Run 'proxy <command> -help' for more information on a command. Run 'proxy <command> -help' for more information on a command.
@ -216,7 +184,6 @@ func runServe() {
databaseURL := fs.String("database-url", "", "PostgreSQL connection URL") databaseURL := fs.String("database-url", "", "PostgreSQL connection URL")
logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error") logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error")
logFormat := fs.String("log-format", "", "Log format: text, json") logFormat := fs.String("log-format", "", "Log format: text, json")
accessLogPath := fs.String("access-log", "", "Path to the JSONL access log")
version := fs.Bool("version", false, "Print version and exit") version := fs.Bool("version", false, "Print version and exit")
fs.Usage = func() { fs.Usage = func() {
@ -234,36 +201,9 @@ func runServe() {
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n") fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
fmt.Fprintf(os.Stderr, " PROXY_ACCESS_LOG_PATH JSONL access log path\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS Comma-separated private upstream hosts\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_LOOPBACK Permit loopback upstreams and redirects\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NPM npm registry upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO Cargo index upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO_DOWNLOAD Cargo download upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GEM RubyGems upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO Go module proxy upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO_VULNDB Go vulnerability database URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX Hex repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX_API Hex API upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PUB pub registry upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI PyPI index and API upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI_DOWNLOAD PyPI download upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET NuGet API upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET_SEARCH NuGet search upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_SWIFT Swift Package Registry upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_OCI_DEFAULT Default OCI registry upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_DEBIAN Debian repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_RPM RPM repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Set false to allow Gradle PUT uploads (default true)\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
@ -316,9 +256,6 @@ func runServe() {
if *logFormat != "" { if *logFormat != "" {
cfg.Log.Format = *logFormat cfg.Log.Format = *logFormat
} }
if *accessLogPath != "" {
cfg.AccessLog.Path = *accessLogPath
}
// Validate configuration // Validate configuration
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
@ -328,13 +265,9 @@ func runServe() {
// Setup logger // Setup logger
logger := setupLogger(cfg.Log.Level, cfg.Log.Format) logger := setupLogger(cfg.Log.Level, cfg.Log.Format)
memlimit.Apply(logger)
// Create and start server // Create and start server
srv, err := server.New(cfg, logger, server.BuildInfo{ srv, err := server.New(cfg, logger)
Version: Version,
Commit: Commit,
})
if err != nil { if err != nil {
logger.Error("failed to create server", "error", err) logger.Error("failed to create server", "error", err)
os.Exit(1) os.Exit(1)
@ -443,8 +376,6 @@ func runMirror() {
sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file") sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version")
fs.Usage = func() { fs.Usage = func() {
fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
@ -452,36 +383,21 @@ func runMirror() {
fmt.Fprintf(os.Stderr, "Examples:\n") fmt.Fprintf(os.Stderr, "Examples:\n")
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n")
fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n")
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n")
fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n")
fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
fmt.Fprintf(os.Stderr, "Flags:\n") fmt.Fprintf(os.Stderr, "Flags:\n")
fs.PrintDefaults() fs.PrintDefaults()
} }
_ = fs.Parse(os.Args[1:]) _ = fs.Parse(os.Args[1:])
purls := fs.Args() purls := fs.Args()
var platformList []string
if *platforms != "" {
platformList = []string{*platforms}
}
if _, err := mirror.ParsePlatforms(platformList); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
// Determine source // Determine source
var source mirror.Source var source mirror.Source
switch { switch {
case *sbomPath != "": case *sbomPath != "":
data, err := os.ReadFile(*sbomPath) source = &mirror.SBOMSource{Path: *sbomPath}
if err != nil {
fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
os.Exit(1)
}
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList}
case len(purls) > 0: case len(purls) > 0:
source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList} source = &mirror.PURLSource{PURLs: purls}
default: default:
fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
fs.Usage() fs.Usage()
@ -495,7 +411,6 @@ func runMirror() {
os.Exit(1) os.Exit(1)
} }
cfg.LoadFromEnv() cfg.LoadFromEnv()
printEnvWarnings(cfg)
if *storageURL != "" { if *storageURL != "" {
cfg.Storage.URL = *storageURL cfg.Storage.URL = *storageURL
@ -514,10 +429,6 @@ func runMirror() {
fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err) fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
os.Exit(1) os.Exit(1)
} }
if !cfg.Storage.CacheArtifacts {
fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n")
os.Exit(1)
}
logger := setupLogger("info", "text") logger := setupLogger("info", "text")
@ -553,27 +464,15 @@ func runMirror() {
os.Exit(1) //nolint:gocritic // db closed above os.Exit(1) //nolint:gocritic // db closed above
} }
// Build proxy with the same upstream clients and authentication as serve // Build proxy (reuses same pipeline as serve)
proxy, images := server.NewMirrorProxy(cfg, db, store, logger) fetcher := fetch.NewFetcher()
resolver := fetch.NewResolver()
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
proxy.CacheMetadata = true // mirror always caches metadata proxy.CacheMetadata = true // mirror always caches metadata
proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataTTL = cfg.ParseMetadataTTL()
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize()
// As serve does, resolve Swift source repositories in the denylist through
// the upstream, and refuse to run when one can't be.
if err := server.ConfigureDenylist(cfg, proxy); err != nil {
_ = db.Close()
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1) //nolint:gocritic // db closed above
}
m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) m := mirror.New(proxy, db, store, logger, *concurrency)
switch s := source.(type) {
case *mirror.PURLSource:
s.Versions = m.Versions
case *mirror.SBOMSource:
s.Versions = m.Versions
}
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
go func() { go func() {
@ -596,9 +495,7 @@ func runMirror() {
return return
} }
closeGit := useCommandGit(cfg, proxy, m, logger)
progress, err := m.Run(ctx, source) progress, err := m.Run(ctx, source)
closeGit()
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err) fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1) os.Exit(1)
@ -613,75 +510,6 @@ func runMirror() {
fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error) fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error)
} }
} }
// A partial mirror has to be visible in the exit status, or a CI step
// that prewarms the cache passes even when every download failed.
if code := mirrorExitCode(progress); code != 0 {
_ = db.Close()
os.Exit(code) //nolint:gocritic // db closed above
}
}
// newCommandMirror builds the mirror proxy mirror runs, which fetches and
// lists versions from the configured upstreams.
func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler,
db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror {
m := mirror.New(proxy, db, store, logger, workers)
m.Images = images
m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI)
m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient)
if cfg.Scanning.Enabled {
refuseUnscannedMirroring(m)
}
return m
}
// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
// when upstream.git is configured, and returns the function that closes it.
// Where it can't be opened, such as while the running server holds
// git.cache_dir, those items fail with the reason and the rest are mirrored.
func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
if err != nil {
m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
return func() {}
}
if git == nil {
return func() {}
}
m.Git = git
return func() {
if err := closeGit(); err != nil {
logger.Warn("closing the git cache", "error", err)
}
}
}
// mirrorExitCode returns the exit status for a finished mirror run: 1 when any
// package version failed to mirror, 0 otherwise.
func mirrorExitCode(progress *mirror.Progress) int {
if progress != nil && progress.Failed > 0 {
return 1
}
return 0
}
// Reasons the mirror command gives for refusing items while scanning is
// enabled.
const (
mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
)
// refuseUnscannedMirroring makes m fail every item instead of caching it.
// Scanners fetch the staged bytes from the running server, which this command
// does not start, so anything it cached would skip the pre-cache scan and
// later be served from the cache unscanned. The mirror API's jobs run inside
// the server with its scanners and are unaffected.
func refuseUnscannedMirroring(m *mirror.Mirror) {
m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason}
m.RefusePackages = mirrorScanningPackagesReason
} }
func printStats(db *database.DB, popular, recent int, asJSON bool) error { func printStats(db *database.DB, popular, recent int, asJSON bool) error {
@ -826,14 +654,6 @@ func formatSize(bytes int64) string {
} }
} }
// printEnvWarnings reports environment values LoadFromEnv ignored. serve
// logs them from server.Start; mirror never gets there.
func printEnvWarnings(cfg *config.Config) {
for _, warning := range cfg.EnvWarnings() {
fmt.Fprintf(os.Stderr, "warning: %s\n", warning)
}
}
func loadConfig(path string) (*config.Config, error) { func loadConfig(path string) (*config.Config, error) {
if path != "" { if path != "" {
return config.Load(path) return config.Load(path)

View file

@ -1,80 +0,0 @@
package main
import (
"os"
"os/exec"
"strings"
"testing"
)
func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) {
cmd := exec.Command(os.Args[0], "-test.run=^TestServeHelpProcess$")
cmd.Env = append(os.Environ(), "PROXY_TEST_SERVE_HELP=1")
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("serve help failed: %v\n%s", err, output)
}
variables := []string{
"PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS",
"PROXY_UPSTREAM_ALLOW_LOOPBACK",
"PROXY_UPSTREAM_NPM",
"PROXY_UPSTREAM_CARGO",
"PROXY_UPSTREAM_CARGO_DOWNLOAD",
"PROXY_UPSTREAM_GEM",
"PROXY_UPSTREAM_GO",
"PROXY_UPSTREAM_HEX",
"PROXY_UPSTREAM_HEX_API",
"PROXY_UPSTREAM_PUB",
"PROXY_UPSTREAM_PYPI",
"PROXY_UPSTREAM_PYPI_DOWNLOAD",
"PROXY_UPSTREAM_MAVEN",
"PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL",
"PROXY_UPSTREAM_NUGET",
"PROXY_UPSTREAM_NUGET_SEARCH",
"PROXY_UPSTREAM_COMPOSER",
"PROXY_UPSTREAM_COMPOSER_REPOSITORY",
"PROXY_UPSTREAM_CONAN",
"PROXY_UPSTREAM_CHEF",
"PROXY_UPSTREAM_CONDA",
"PROXY_UPSTREAM_CRAN",
"PROXY_UPSTREAM_JULIA",
"PROXY_UPSTREAM_OCI_DEFAULT",
"PROXY_UPSTREAM_DEBIAN",
"PROXY_UPSTREAM_RPM",
}
for _, variable := range variables {
if !strings.Contains(string(output), variable) {
t.Errorf("serve help omitted %s", variable)
}
}
}
func TestServeHelpProcess(*testing.T) {
if os.Getenv("PROXY_TEST_SERVE_HELP") != "1" {
return
}
os.Args = []string{"proxy", "serve", "-help"}
main()
}
func TestMirrorLogsEnvWarnings(t *testing.T) {
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$")
// The negative size stops mirror before it opens any storage.
cmd.Env = append(os.Environ(),
"PROXY_TEST_MIRROR_ENV_WARNINGS=1",
"PROXY_CACHE_METADATA=maybe",
"PROXY_STORAGE_MAX_SIZE=-1")
output, _ := cmd.CombinedOutput()
if !strings.Contains(string(output), "PROXY_CACHE_METADATA") {
t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output)
}
}
func TestMirrorEnvWarningsProcess(*testing.T) {
if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" {
return
}
os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"}
main()
}

View file

@ -1,132 +0,0 @@
package main
import (
"net/http"
"net/http/cgi"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/git-pkgs/proxy/internal/gitcache"
)
// gitMirrorUpstream serves team/repo, with one commit on main, from git
// http-backend, and returns the server and the commit.
func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
t.Helper()
git := func(dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
}
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
}
return strings.TrimSpace(string(out))
}
backend := ""
for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
backend = path
}
}
if backend == "" {
t.Skip("git http-backend is not installed")
}
root := t.TempDir()
bare := filepath.Join(root, "team", "repo.git")
git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
work := t.TempDir()
git("", "clone", "--quiet", bare, work)
git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
git(work, "push", "--quiet", "origin", "HEAD:main")
head := git(work, "rev-parse", "HEAD")
server := httptest.NewServer(&cgi.Handler{
Path: backend,
Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
})
t.Cleanup(server.Close)
return server, head
}
func fileExistsForTest(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// writeGitMirrorConfig writes a config with upstream.git "up" at
// upstreamURL and the git cache, storage and database under dir.
func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
t.Helper()
cfg := "storage:\n" +
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
"database:\n" +
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
"upstream:\n" +
" allow_loopback: true\n" +
" git:\n" +
" up: " + upstreamURL + "\n" +
"git:\n" +
" cache_dir: " + filepath.Join(dir, "git") + "\n"
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
t.Fatalf("writing config: %v", err)
}
return path
}
func TestMirrorCommandCachesGitCommit(t *testing.T) {
upstream, head := gitMirrorUpstream(t)
dir := t.TempDir()
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
output := runMirrorProcess(t, 0, "-config", configPath, purl)
if !strings.Contains(output, "1 downloaded") {
t.Errorf("output:\n%s", output)
}
cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
if len(cached) != 1 {
t.Errorf("cached repositories %v, want team/repo.git", cached)
}
output = runMirrorProcess(t, 0, "-config", configPath, purl)
if !strings.Contains(output, "1 skipped") {
t.Errorf("second run output:\n%s", output)
}
}
// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
// mirror API, while the running server holds git.cache_dir.
func TestMirrorCommandGitCacheInUse(t *testing.T) {
upstream, head := gitMirrorUpstream(t)
dir := t.TempDir()
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
cache, err := gitcache.New(gitcache.Options{
CacheDir: filepath.Join(dir, "git"),
RefreshWait: time.Second,
Client: &http.Client{},
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
output := runMirrorProcess(t, 1, "-config", configPath, purl)
for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
if !strings.Contains(output, want) {
t.Errorf("output missing %q:\n%s", want, output)
}
}
}

View file

@ -1,186 +0,0 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"github.com/git-pkgs/proxy/internal/database"
)
const swiftWidgetArchive = "acme widget archive"
// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package
// registry does, with https://github.com/acme/widget as its source
// repository, and counts archive downloads and identifier lookups. The
// repository's identifiers are ids, or only acme.widget.
func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) {
t.Helper()
if len(ids) == 0 {
ids = []string{"acme.widget"}
}
identifiers, err := json.Marshal(map[string][]string{"identifiers": ids})
if err != nil {
t.Fatal(err)
}
downloads, lookups = new(atomic.Int64), new(atomic.Int64)
sum := sha256.Sum256([]byte(swiftWidgetArchive))
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Version", "1")
switch r.URL.Path {
case "/identifiers":
lookups.Add(1)
if r.URL.Query().Get("url") != "https://github.com/acme/widget" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(identifiers)
case "/acme/widget":
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`)
case "/acme/widget/1.0.0/Package.swift":
w.Header().Set("Content-Type", "text/x-swift")
_, _ = io.WriteString(w, "// swift-tools-version:5.9\n")
case "/acme/widget/1.0.0":
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:]))
case "/acme/widget/1.0.0.zip":
downloads.Add(1)
w.Header().Set("Content-Type", "application/zip")
_, _ = io.WriteString(w, swiftWidgetArchive)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv, downloads, lookups
}
// writeSwiftMirrorConfig writes a config using upstream as the Swift
// registry, denying the denied package PURLs.
func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string {
t.Helper()
cfg := "storage:\n" +
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
"database:\n" +
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
"upstream:\n" +
" swift: " + upstream + "\n" +
" allow_loopback: true\n"
if len(denied) > 0 {
cfg += "denylist:\n packages:\n"
for _, p := range denied {
cfg += " - " + p + "\n"
}
}
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
t.Fatalf("writing config: %v", err)
}
return path
}
func cachedArtifacts(t *testing.T, dir string) int64 {
t.Helper()
db, err := database.Open(filepath.Join(dir, "proxy.db"))
if err != nil {
t.Fatalf("opening database: %v", err)
}
defer func() { _ = db.Close() }()
stats, err := db.GetCacheStats()
if err != nil {
t.Fatalf("GetCacheStats: %v", err)
}
return stats.TotalArtifacts
}
// Without a denylist the fake registry mirrors the release, so the tests
// below fail for the denylist and not for the setup.
func TestMirrorSwiftRelease(t *testing.T) {
upstream, downloads, _ := swiftMirrorUpstream(t)
dir := t.TempDir()
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL)
output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
if !strings.Contains(output, "1 downloaded") {
t.Errorf("output:\n%s", output)
}
if n := downloads.Load(); n != 1 {
t.Errorf("archive downloads = %d, want 1", n)
}
if n := cachedArtifacts(t, dir); n != 1 {
t.Errorf("cached artifacts = %d, want 1", n)
}
}
// A denylist entry naming the package by its source repository blocks the
// mirror download of the registry identity it resolves to.
func TestMirrorDenylistSwiftSourceEntry(t *testing.T) {
upstream, downloads, _ := swiftMirrorUpstream(t)
dir := t.TempDir()
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
t.Errorf("output:\n%s", output)
}
if n := downloads.Load(); n != 0 {
t.Errorf("archive downloads = %d, want 0", n)
}
if n := cachedArtifacts(t, dir); n != 0 {
t.Errorf("cached artifacts = %d, want 0", n)
}
}
// The registry may list several identities for one repository. A source
// entry denies all of them, not only the first.
func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) {
upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget")
dir := t.TempDir()
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
t.Errorf("output:\n%s", output)
}
if n := downloads.Load(); n != 0 {
t.Errorf("archive downloads = %d, want 0", n)
}
}
// A source repository entry the registry can't resolve stops the mirror
// before it downloads anything, as it stops serve: left out, it would let
// the version through.
func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) {
upstream, downloads, lookups := swiftMirrorUpstream(t)
dir := t.TempDir()
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0")
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") {
t.Errorf("output does not report the unresolved entry:\n%s", output)
}
if strings.Contains(output, "Mirror complete") {
t.Errorf("mirror ran:\n%s", output)
}
if lookups.Load() == 0 {
t.Error("the entry was not looked up")
}
if n := downloads.Load(); n != 0 {
t.Errorf("archive downloads = %d, want 0", n)
}
}

View file

@ -1,214 +0,0 @@
package main
import (
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/mirror"
)
// mirrorTestEnv carries the mirror command's arguments into the helper
// process, one per line.
const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS"
// writeScanningMirrorConfig writes a config with scanning enabled and storage
// and database under dir. The scanner address is never reached.
func writeScanningMirrorConfig(t *testing.T, dir string) string {
t.Helper()
cfg := "storage:\n" +
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
"database:\n" +
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
"scanning:\n" +
" enabled: true\n" +
" signing_key: test-signing-key\n" +
" scanners:\n" +
" - name: test\n" +
" url: http://127.0.0.1:1/scan\n"
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
t.Fatalf("writing config: %v", err)
}
return path
}
// runMirrorProcess runs proxy mirror with args in a helper process and
// checks that it exits with wantCode.
func runMirrorProcess(t *testing.T, wantCode int, args ...string) string {
t.Helper()
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$")
cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n"))
output, err := cmd.CombinedOutput()
code := 0
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
code = exitErr.ExitCode()
} else if err != nil {
t.Fatalf("proxy mirror %v: %v\n%s", args, err, output)
}
if code != wantCode {
t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output)
}
return string(output)
}
// With scanning enabled, proxy mirror refuses packages rather than caching
// them unscanned: scanners fetch staged bytes from the running server, which
// the command does not start.
// A refused package counts as failed, so the command exits 1.
func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) {
dir := t.TempDir()
configPath := writeScanningMirrorConfig(t, dir)
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21")
for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} {
if !strings.Contains(output, want) {
t.Errorf("output missing %q:\n%s", want, output)
}
}
db, err := database.Open(filepath.Join(dir, "proxy.db"))
if err != nil {
t.Fatalf("opening database: %v", err)
}
defer func() { _ = db.Close() }()
stats, err := db.GetCacheStats()
if err != nil {
t.Fatalf("GetCacheStats: %v", err)
}
if stats.TotalArtifacts != 0 {
t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts)
}
var stored []string
_ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error {
if err == nil && !d.IsDir() {
stored = append(stored, path)
}
return nil
})
if len(stored) != 0 {
t.Errorf("files written to storage: %v", stored)
}
}
// A dry run caches nothing, so it still lists what would be mirrored.
func TestMirrorCommandDryRunWhileScanning(t *testing.T) {
configPath := writeScanningMirrorConfig(t, t.TempDir())
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21")
if !strings.Contains(output, "Would mirror 1 package versions") {
t.Errorf("dry run output:\n%s", output)
}
if strings.Contains(output, mirrorScanningPackagesReason) {
t.Errorf("dry run reported the refusal:\n%s", output)
}
}
func TestMirrorCommandSBOMComponentVersion(t *testing.T) {
dir := t.TempDir()
configPath := writeScanningMirrorConfig(t, dir)
sbomPath := filepath.Join(dir, "bom.json")
data := []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`)
if err := os.WriteFile(sbomPath, data, 0600); err != nil {
t.Fatal(err)
}
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "-sbom", sbomPath)
for _, want := range []string{"Would mirror 1 package versions", "pkg:docker/ghcr.io/team/app@1.0"} {
if !strings.Contains(output, want) {
t.Errorf("dry run missing %q:\n%s", want, output)
}
}
}
func TestRefuseUnscannedMirroring(t *testing.T) {
m := mirror.New(nil, nil, nil, nil, 1)
refuseUnscannedMirroring(m)
if m.RefusePackages != mirrorScanningPackagesReason {
t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason)
}
images, ok := m.Images.(mirror.UnavailableImages)
if !ok || images.Reason != mirrorScanningImagesReason {
t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images)
}
}
func TestMirrorHelperProcess(*testing.T) {
args := os.Getenv(mirrorTestEnv)
if args == "" {
return
}
os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...)
main()
}
func TestMirrorExitCode(t *testing.T) {
tests := []struct {
name string
progress *mirror.Progress
want int
}{
{"nil progress", nil, 0},
{"everything mirrored", &mirror.Progress{Total: 2, Completed: 2}, 0},
{"everything cached", &mirror.Progress{Total: 2, Skipped: 2}, 0},
{"one failure", &mirror.Progress{Total: 2, Completed: 1, Failed: 1}, 1},
{"all failed", &mirror.Progress{Total: 3, Failed: 3}, 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := mirrorExitCode(tt.progress); got != tt.want {
t.Errorf("mirrorExitCode() = %d, want %d", got, tt.want)
}
})
}
}
// proxy mirror used to exit 0 after printing "N failed", so a CI step that
// prewarms the cache passed even when every download failed (#500). An
// ecosystem the resolver rejects fails without touching the network.
func TestMirrorExitsNonZeroWhenDownloadsFail(t *testing.T) {
dir := t.TempDir()
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorFailureProcess$")
cmd.Env = append(os.Environ(),
"PROXY_TEST_MIRROR_FAILURE=1",
"PROXY_TEST_MIRROR_DIR="+dir,
)
output, err := cmd.CombinedOutput()
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatalf("mirror with a failed download exited with %v, want status 1\n%s", err, output)
}
if code := exitErr.ExitCode(); code != 1 {
t.Fatalf("exit status = %d, want 1\n%s", code, output)
}
if !strings.Contains(string(output), "1 failed") {
t.Errorf("output does not report the failure:\n%s", output)
}
if !strings.Contains(string(output), "bogus/foo@1.0.0") {
t.Errorf("output does not list the failed package:\n%s", output)
}
}
func TestMirrorFailureProcess(*testing.T) {
if os.Getenv("PROXY_TEST_MIRROR_FAILURE") != "1" {
return
}
dir := os.Getenv("PROXY_TEST_MIRROR_DIR")
os.Args = []string{
"proxy", "mirror",
"--storage-url", "file://" + filepath.Join(dir, "store"),
"--database-path", filepath.Join(dir, "cache.db"),
"pkg:bogus/foo@1.0.0",
}
main()
}

Some files were not shown because too many files have changed in this diff Show more