mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-09 07:39:19 -05:00
Compare commits
561 changed files with 5995 additions and 124097 deletions
|
|
@ -1,17 +0,0 @@
|
|||
# Version and commit come in as build args, so the build doesn't need .git.
|
||||
.git
|
||||
.github
|
||||
|
||||
# Local build output and runtime data
|
||||
/proxy
|
||||
dist/
|
||||
build/
|
||||
cache/
|
||||
*.db
|
||||
*.tar.gz
|
||||
|
||||
# Local config and secrets
|
||||
.env
|
||||
.env.local
|
||||
proxy.cfg
|
||||
config.yaml
|
||||
90
.github/workflows/ci.yml
vendored
90
.github/workflows/ci.yml
vendored
|
|
@ -13,17 +13,18 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
go-version: ['1.25']
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
- name: Build
|
||||
run: go build -v ./...
|
||||
|
|
@ -31,92 +32,17 @@ jobs:
|
|||
- name: Test
|
||||
run: go test -v -race ./...
|
||||
|
||||
postgres:
|
||||
name: test (postgres)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:b8faeff7712663c2773a190181e1246e37e364b887611c04658c28b44e2408fe
|
||||
env:
|
||||
POSTGRES_PASSWORD: test
|
||||
POSTGRES_DB: proxy
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d proxy"
|
||||
--health-interval 2s
|
||||
--health-timeout 5s
|
||||
--health-retries 30
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
# The packages with tests that need PROXY_DATABASE_URL. -p 1 because
|
||||
# their tests drop and recreate tables in the one shared database.
|
||||
- name: Test
|
||||
env:
|
||||
PROXY_DATABASE_URL: postgres://postgres:test@localhost:5432/proxy?sslmode=disable
|
||||
run: go test -v -race -p 1 ./internal/database/... ./internal/handler/... ./internal/server/...
|
||||
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0
|
||||
go-version: '1.25'
|
||||
|
||||
- name: golangci-lint
|
||||
run: golangci-lint run ./...
|
||||
|
||||
helm:
|
||||
name: Helm chart
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
|
||||
with:
|
||||
version: v3.18.6
|
||||
|
||||
- name: Lint chart
|
||||
run: helm lint deploy/charts/proxy
|
||||
|
||||
- name: Render chart variants
|
||||
run: |
|
||||
set -euo pipefail
|
||||
helm template proxy deploy/charts/proxy >/dev/null
|
||||
helm template proxy deploy/charts/proxy \
|
||||
--set persistence.enabled=false \
|
||||
--set config.existingConfigMap=proxy-config \
|
||||
--set ingress.enabled=true \
|
||||
--set 'ingress.hosts[0].host=proxy.example.com' \
|
||||
--set 'ingress.hosts[0].paths[0].path=/' \
|
||||
--set 'ingress.hosts[0].paths[0].pathType=Prefix' \
|
||||
>/dev/null
|
||||
helm template proxy deploy/charts/proxy \
|
||||
--set 'extraVolumes[0].name=corp-ca' \
|
||||
--set 'extraVolumes[0].configMap.name=corp-ca' \
|
||||
--set 'extraVolumeMounts[0].name=corp-ca' \
|
||||
--set 'extraVolumeMounts[0].mountPath=/etc/proxy/ca' \
|
||||
--set 'extraVolumeMounts[0].readOnly=true' \
|
||||
--set 'extraInitContainers[0].name=wait-for-db' \
|
||||
--set 'extraInitContainers[0].image=busybox:1.37' \
|
||||
--set 'extraInitContainers[0].command={sh,-c,exit 0}' \
|
||||
>/dev/null
|
||||
run: go tool golangci-lint run ./...
|
||||
|
|
|
|||
137
.github/workflows/monthly-release.yml
vendored
137
.github/workflows/monthly-release.yml
vendored
|
|
@ -1,137 +0,0 @@
|
|||
name: Monthly release
|
||||
|
||||
# Cuts a release on the 1st of each month when main is green and something
|
||||
# has merged since the last release: a patch release, or a new minor version
|
||||
# if a pull request labelled "breaking" was merged (a new major version from
|
||||
# 1.0 on). A breaking release is created as a draft, and its images and chart
|
||||
# are published only when someone publishes the draft, after writing the
|
||||
# upgrade notes. Scheduled runs only report what they would do until the
|
||||
# repository variable MONTHLY_RELEASE is "true".
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 1 * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Only report what would be released
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: monthly-release
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
checks: read
|
||||
pull-requests: read
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Decide whether to release
|
||||
id: plan
|
||||
env:
|
||||
EVENT: ${{ github.event_name }}
|
||||
INPUT_DRY_RUN: ${{ inputs.dry_run }}
|
||||
ENABLED: ${{ vars.MONTHLY_RELEASE }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$EVENT" = workflow_dispatch ]; then
|
||||
dry_run=$INPUT_DRY_RUN
|
||||
elif [ "$ENABLED" = true ]; then
|
||||
dry_run=false
|
||||
else
|
||||
dry_run=true
|
||||
fi
|
||||
echo "dry_run=$dry_run" >> "$GITHUB_OUTPUT"
|
||||
|
||||
sha=$(git rev-parse HEAD)
|
||||
last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*')
|
||||
# A breaking release waits as a draft for its upgrade notes. Building
|
||||
# on its tag would publish those changes without them. Older tags
|
||||
# were released by hand and may have no release at all.
|
||||
draft=$(gh release view "$last" -R "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo false)
|
||||
if [ "$draft" = true ]; then
|
||||
echo "::warning::$last is still a draft, publish it first"
|
||||
exit 0
|
||||
fi
|
||||
merged=$(git rev-list --count "$last..$sha")
|
||||
if [ "$merged" -eq 0 ]; then
|
||||
echo "Nothing merged since $last"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Every check on main's HEAD must have finished green. This run
|
||||
# shows up as a check on the same commit, so it is left out.
|
||||
bad=$(gh api --paginate "repos/$REPO/commits/$sha/check-runs?per_page=100" \
|
||||
--jq ".check_runs[]
|
||||
| select(.html_url | contains(\"/actions/runs/$GITHUB_RUN_ID/\") | not)
|
||||
| select(.status != \"completed\"
|
||||
or (.conclusion | IN(\"success\", \"skipped\", \"neutral\") | not))
|
||||
| .name")
|
||||
if [ -n "$bad" ]; then
|
||||
echo "::warning::main ($sha) is not green, skipping: $(echo "$bad" | sort -u | paste -sd, -)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
since=$(git log -1 --format=%cI "$last")
|
||||
breaking=$(gh pr list -R "$REPO" --state merged --base main --limit 1000 \
|
||||
--search "merged:>$since label:breaking" --json number --jq '.[].number')
|
||||
|
||||
IFS=. read -r major minor patch <<<"${last#v}"
|
||||
if [ -z "$breaking" ]; then
|
||||
tag="v$major.$minor.$((patch + 1))"
|
||||
else
|
||||
echo "Breaking changes merged since $last: #$(echo "$breaking" | sort -n | paste -sd, - | sed 's/,/, #/g')"
|
||||
if [ "$major" -eq 0 ]; then
|
||||
tag="v0.$((minor + 1)).0"
|
||||
else
|
||||
tag="v$((major + 1)).0.0"
|
||||
fi
|
||||
fi
|
||||
echo "$merged commits since $last, next release $tag at $sha"
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "draft=$([ -n "$breaking" ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the
|
||||
# release workflows are dispatched on the new tag instead.
|
||||
- name: Tag and start the release workflows
|
||||
if: steps.plan.outputs.tag != ''
|
||||
env:
|
||||
TAG: ${{ steps.plan.outputs.tag }}
|
||||
SHA: ${{ steps.plan.outputs.sha }}
|
||||
DRY_RUN: ${{ steps.plan.outputs.dry_run }}
|
||||
DRAFT: ${{ steps.plan.outputs.draft }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$DRAFT" = true ]; then
|
||||
what="start release.yml for a draft release"
|
||||
else
|
||||
what="start release.yml and publish.yml"
|
||||
fi
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "Dry run: would tag $TAG at $SHA and $what"
|
||||
exit 0
|
||||
fi
|
||||
gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null
|
||||
gh workflow run release.yml -R "$REPO" --ref "$TAG" -f draft="$DRAFT"
|
||||
if [ "$DRAFT" = true ]; then
|
||||
echo "::notice::$TAG is a draft release: add the upgrade notes and publish it to push the images and chart"
|
||||
else
|
||||
gh workflow run publish.yml -R "$REPO" --ref "$TAG"
|
||||
fi
|
||||
echo "Tagged $TAG at $SHA"
|
||||
130
.github/workflows/publish.yml
vendored
130
.github/workflows/publish.yml
vendored
|
|
@ -5,10 +5,6 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
# A draft release made by monthly-release.yml is published by hand. Releases
|
||||
# published with GITHUB_TOKEN don't start this.
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions: {}
|
||||
|
||||
|
|
@ -22,23 +18,14 @@ jobs:
|
|||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
|
@ -46,123 +33,14 @@ jobs:
|
|||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository }}
|
||||
|
||||
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ github.sha }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
|
||||
- name: Sign image by digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
cosign sign --yes "${IMAGE}@${DIGEST}"
|
||||
|
||||
- name: Verify BuildKit attestations and extract SPDX predicates
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
reference="${IMAGE}@${DIGEST}"
|
||||
docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json
|
||||
docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json
|
||||
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \
|
||||
provenance.json >/dev/null
|
||||
jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json"
|
||||
done
|
||||
|
||||
- name: Attest platform SBOMs by digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
reference="${IMAGE}@${DIGEST}"
|
||||
for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do
|
||||
cosign attest --yes --type spdxjson --predicate "$predicate" "$reference"
|
||||
done
|
||||
|
||||
publish_chart:
|
||||
name: Push Helm chart to GHCR
|
||||
if: github.ref_type == 'tag'
|
||||
needs: push_to_registry
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
- uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
|
||||
with:
|
||||
version: v3.18.6
|
||||
|
||||
- name: Validate and normalize release version
|
||||
id: version
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
semver='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?$'
|
||||
[[ "$TAG" =~ $semver ]] || {
|
||||
echo "Tag must be strict SemVer of the form vMAJOR.MINOR.PATCH[-PRERELEASE]: $TAG" >&2
|
||||
exit 1
|
||||
}
|
||||
version="${TAG#v}"
|
||||
[[ "$version" != "0.0.0" ]] || {
|
||||
echo "0.0.0 is a development placeholder and must not be published" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Log in to GHCR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: printf '%s' "$GH_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
|
||||
|
||||
- name: Lint and package chart
|
||||
env:
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
helm lint deploy/charts/proxy
|
||||
mkdir -p build
|
||||
helm package \
|
||||
--destination build \
|
||||
--version "$VERSION" \
|
||||
--app-version "$VERSION" \
|
||||
deploy/charts/proxy
|
||||
metadata="$(helm show chart "build/proxy-${VERSION}.tgz")"
|
||||
[[ "$(awk '$1 == "version:" {print $2}' <<<"$metadata")" == "$VERSION" ]]
|
||||
[[ "$(awk '$1 == "appVersion:" {gsub(/\"/, "", $2); print $2}' <<<"$metadata")" == "$VERSION" ]]
|
||||
|
||||
- name: Push chart
|
||||
env:
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
run: helm push "build/proxy-${VERSION}.tgz" oci://ghcr.io/git-pkgs/charts
|
||||
|
|
|
|||
16
.github/workflows/release.yml
vendored
16
.github/workflows/release.yml
vendored
|
|
@ -4,13 +4,6 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
# Started by monthly-release.yml on the tag it creates.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
draft:
|
||||
description: Create the release as a draft
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
|
@ -18,11 +11,10 @@ permissions:
|
|||
|
||||
jobs:
|
||||
release:
|
||||
if: github.ref_type == 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
|
@ -30,14 +22,14 @@ jobs:
|
|||
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0
|
||||
- uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2
|
||||
with:
|
||||
version: "~> v2"
|
||||
args: release --clean${{ inputs.draft && ' --draft' || '' }}
|
||||
args: release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
|
|||
8
.github/workflows/swagger.yml
vendored
8
.github/workflows/swagger.yml
vendored
|
|
@ -12,17 +12,17 @@ jobs:
|
|||
swagger:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Install swag
|
||||
run: go install github.com/swaggo/swag/cmd/swag@v1.16.6
|
||||
run: go install github.com/swaggo/swag/cmd/swag@latest
|
||||
|
||||
- name: Generate swagger
|
||||
run: go generate ./internal/server
|
||||
|
|
|
|||
20
.github/workflows/zizmor.yml
vendored
20
.github/workflows/zizmor.yml
vendored
|
|
@ -5,22 +5,14 @@ on:
|
|||
branches:
|
||||
- main
|
||||
paths:
|
||||
- '.github/**'
|
||||
- '**/action.yml'
|
||||
- '**/action.yaml'
|
||||
- 'zizmor.yml'
|
||||
- 'zizmor.yaml'
|
||||
- '.github/workflows/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- '.github/**'
|
||||
- '**/action.yml'
|
||||
- '**/action.yaml'
|
||||
- 'zizmor.yml'
|
||||
- 'zizmor.yaml'
|
||||
- '.github/workflows/**'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
zizmor:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -29,9 +21,9 @@ jobs:
|
|||
security-events: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0
|
||||
uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7
|
||||
|
|
|
|||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -4,7 +4,7 @@
|
|||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
/proxy
|
||||
proxy
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
|
@ -43,4 +43,4 @@ cache/*
|
|||
|
||||
# Debug files
|
||||
__debug_bin
|
||||
debug
|
||||
debug
|
||||
|
|
@ -26,7 +26,3 @@ linters:
|
|||
- goconst
|
||||
- dupl
|
||||
- mnd
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ builds:
|
|||
- linux
|
||||
- darwin
|
||||
- windows
|
||||
- freebsd
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ proxy/
|
|||
│ │ └── queries.go # CRUD operations
|
||||
│ ├── storage/ # Artifact file storage
|
||||
│ │ ├── storage.go # Storage interface
|
||||
│ │ └── blob.go # gocloud.dev/blob backends (file, S3, Azure)
|
||||
│ │ └── filesystem.go # Local filesystem impl
|
||||
│ ├── upstream/ # Upstream registry clients
|
||||
│ │ ├── fetcher.go # HTTP artifact fetching
|
||||
│ │ └── resolver.go # Download URL resolution
|
||||
|
|
@ -72,7 +72,7 @@ Key types:
|
|||
|
||||
### `internal/storage`
|
||||
|
||||
Artifact file storage abstraction backed by `gocloud.dev/blob`. Supports local filesystem (`file://`), S3 (`s3://`), and Azure (`azblob://`) URLs.
|
||||
Artifact file storage abstraction. Currently implements local filesystem storage. Designed to allow future backends (S3, GCS).
|
||||
|
||||
Interface:
|
||||
```go
|
||||
|
|
@ -160,7 +160,6 @@ mux.Handle("/newregistry/", http.StripPrefix("/newregistry", newHandler.Routes()
|
|||
- Keep functions short and focused
|
||||
- Write tests for new functionality
|
||||
- Document exported types and functions
|
||||
- A new Prometheus metric needs a tile on `/ui/analytics` and an entry in `metricSurface` (`internal/server/analytics_coverage_test.go`). The page is meant to be a complete view of `/metrics`, so `TestEveryMetricIsSurfaced` fails until both exist.
|
||||
|
||||
## Testing
|
||||
|
||||
|
|
@ -212,24 +211,6 @@ Add PyPI registry support
|
|||
- Add tests for wheel and sdist downloads
|
||||
```
|
||||
|
||||
## Reviewing and Merging
|
||||
|
||||
Maintainers are the members of the proxy team in the git-pkgs org. Any maintainer can review and merge. A pull request needs green CI and an approval from a maintainer other than its author. A "request changes" review from any maintainer blocks it until that reviewer approves or withdraws the review.
|
||||
|
||||
Using AI to write or review is fine, but whoever approves a pull request should understand it and stand behind it. If it touches an ecosystem you don't use, you can ask someone who does to look before you approve.
|
||||
|
||||
Open an issue before a bigger change: anything that changes behaviour users rely on, new configuration that would be hard to remove, or a change that fans out into other git-pkgs modules. If no maintainer objects within a few days, open the pull request; it's reviewed like any other. Objections are worked out in the issue, with @andrew breaking ties for now. Where possible, deprecate before removing: an old configuration key keeps working, with a warning, in at least one release before the one that removes it.
|
||||
|
||||
## Releases
|
||||
|
||||
A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on.
|
||||
|
||||
A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. Until then the monthly run releases nothing.
|
||||
|
||||
Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default.
|
||||
|
||||
To release by hand, push a `v*` tag.
|
||||
|
||||
## Questions?
|
||||
|
||||
Open an issue on GitHub if you have questions or need help getting started.
|
||||
|
|
|
|||
19
Dockerfile
19
Dockerfile
|
|
@ -1,4 +1,4 @@
|
|||
FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS builder
|
||||
FROM golang:1.26.4-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
|
|
@ -12,17 +12,12 @@ RUN go mod download
|
|||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the binary for the target platform
|
||||
ARG TARGETARCH
|
||||
ARG VERSION=dev
|
||||
ARG COMMIT=unknown
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
|
||||
-ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \
|
||||
-o /proxy ./cmd/proxy
|
||||
# Build the binary
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||
|
||||
FROM alpine:3.24.3
|
||||
FROM alpine:3.24.1
|
||||
|
||||
RUN apk add --no-cache ca-certificates git
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY --from=builder /proxy /usr/local/bin/proxy
|
||||
|
||||
|
|
@ -35,9 +30,5 @@ WORKDIR /data
|
|||
|
||||
EXPOSE 8080
|
||||
|
||||
# /health checks the database and storage and can take up to ten seconds.
|
||||
HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1
|
||||
|
||||
ENTRYPOINT ["proxy"]
|
||||
CMD ["serve"]
|
||||
|
|
|
|||
|
|
@ -1,278 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/handler"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultProxyURL = "http://localhost:8080"
|
||||
hostedTokenEnv = "PROXY_HOSTED_TOKEN"
|
||||
hostedClientTimeout = 10 * time.Minute
|
||||
|
||||
pushArgCount = 4 // ecosystem, name, version, file
|
||||
listArgCount = 2 // ecosystem, name
|
||||
deleteArgCount = 3 // ecosystem, name, version
|
||||
deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files
|
||||
tablePadding = 2
|
||||
)
|
||||
|
||||
// hostedArtifact mirrors the JSON the /api/hosted endpoints return.
|
||||
type hostedArtifact struct {
|
||||
Version string `json:"version"`
|
||||
Filename string `json:"filename"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
PushedAt time.Time `json:"pushed_at"`
|
||||
}
|
||||
|
||||
type hostedPushResponse struct {
|
||||
Ecosystem string `json:"ecosystem"`
|
||||
Name string `json:"name"`
|
||||
hostedArtifact
|
||||
}
|
||||
|
||||
type hostedListResponse struct {
|
||||
Ecosystem string `json:"ecosystem"`
|
||||
Name string `json:"name"`
|
||||
Artifacts []hostedArtifact `json:"artifacts"`
|
||||
}
|
||||
|
||||
type hostedCommand struct {
|
||||
flags *flag.FlagSet
|
||||
proxyURL *string
|
||||
usage string
|
||||
}
|
||||
|
||||
func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand {
|
||||
fs := flag.NewFlagSet(name, flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
proxyURL := getenv("PROXY_URL")
|
||||
if proxyURL == "" {
|
||||
proxyURL = defaultProxyURL
|
||||
}
|
||||
cmd := hostedCommand{flags: fs, usage: usage}
|
||||
cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage)
|
||||
fs.SetOutput(os.Stderr)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
// parse parses flags and checks for between minArgs and maxArgs positional
|
||||
// arguments.
|
||||
func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) {
|
||||
if err := c.flags.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
c.flags.Usage()
|
||||
}
|
||||
return nil, fmt.Errorf("%w\nusage: %s", err, c.usage)
|
||||
}
|
||||
if n := c.flags.NArg(); n < minArgs || n > maxArgs {
|
||||
return nil, fmt.Errorf("usage: %s", c.usage)
|
||||
}
|
||||
return c.flags.Args(), nil
|
||||
}
|
||||
|
||||
func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string {
|
||||
segments := []string{url.PathEscape(ecosystem)}
|
||||
for segment := range strings.SplitSeq(name, "/") {
|
||||
segments = append(segments, url.PathEscape(segment))
|
||||
}
|
||||
for _, v := range version {
|
||||
segments = append(segments, url.PathEscape(v))
|
||||
}
|
||||
return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
func hostedToken(getenv func(string) string) (string, error) {
|
||||
token := getenv(hostedTokenEnv)
|
||||
if token == "" {
|
||||
return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv)
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("push", "proxy hosted push [-url URL] <ecosystem> <name> <version> <file>",
|
||||
"Push a package archive to the proxy", getenv)
|
||||
positional, err := cmd.parse(args, pushArgCount, pushArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, err := hostedToken(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3]
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
// The proxy checks the upload against this, so a truncated or corrupted
|
||||
// transfer is refused rather than stored.
|
||||
digest := sha256.New()
|
||||
size, err := io.Copy(digest, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
segments := []string{version}
|
||||
if handler.HostedFileInPath(ecosystem) {
|
||||
segments = append(segments, filepath.Base(path))
|
||||
}
|
||||
endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil))
|
||||
req, err := http.NewRequest(http.MethodPut, endpoint, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
// The proxy refuses an existing file or a bad token before reading the
|
||||
// body, so the file is only uploaded once it would be accepted.
|
||||
req.Header.Set("Expect", "100-continue")
|
||||
|
||||
var pushed hostedPushResponse
|
||||
if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n",
|
||||
pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256)
|
||||
return err
|
||||
}
|
||||
|
||||
func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("list", "proxy hosted list [-url URL] <ecosystem> <name>", "List pushed package archives", getenv)
|
||||
positional, err := cmd.parse(args, listArgCount, listArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var listed hostedListResponse
|
||||
if err := doHosted(client, req, http.StatusOK, &listed); err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0)
|
||||
_, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED")
|
||||
for _, a := range listed.Artifacts {
|
||||
_, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339))
|
||||
}
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] <ecosystem> <name> <version> [file]",
|
||||
"Delete a pushed version, or one generic file of it", getenv)
|
||||
positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) {
|
||||
return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage)
|
||||
}
|
||||
token, err := hostedToken(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if err := doHosted(client, req, http.StatusNoContent, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
deleted := positional[1] + "@" + positional[2]
|
||||
if len(positional) > deleteArgCount {
|
||||
deleted += " " + positional[deleteArgCount]
|
||||
}
|
||||
_, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted)
|
||||
return err
|
||||
}
|
||||
|
||||
// doHosted sends req and decodes a successful response into out. Any other
|
||||
// status is returned as an error carrying the server's message.
|
||||
func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error {
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != wantStatus {
|
||||
var apiErr struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" {
|
||||
return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status)
|
||||
}
|
||||
return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
const hostedUsage = "usage: proxy hosted <push|list|delete> [flags] ...\n\n" +
|
||||
" push Push a package archive to the proxy\n" +
|
||||
" list List pushed package archives\n" +
|
||||
" delete Delete a pushed version, or one generic file of it"
|
||||
|
||||
type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error
|
||||
|
||||
var hostedSubcommands = map[string]hostedSubcommand{
|
||||
"push": runPush,
|
||||
"list": runList,
|
||||
"delete": runDelete,
|
||||
}
|
||||
|
||||
// runHosted runs the hosted subcommand named by args[0].
|
||||
func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(hostedUsage)
|
||||
}
|
||||
run, ok := hostedSubcommands[args[0]]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage)
|
||||
}
|
||||
return run(args[1:], getenv, stdout, client)
|
||||
}
|
||||
|
||||
// runHostedCommand runs proxy hosted against the proxy's /api/hosted
|
||||
// endpoints and exits non-zero on failure.
|
||||
func runHostedCommand() {
|
||||
client := &http.Client{Timeout: hostedClientTimeout}
|
||||
if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recordedRequest struct {
|
||||
method, path, query, authorization, body, expect string
|
||||
contentLength int64
|
||||
}
|
||||
|
||||
func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) {
|
||||
t.Helper()
|
||||
recorded := &recordedRequest{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
*recorded = recordedRequest{
|
||||
method: r.Method,
|
||||
path: r.URL.EscapedPath(),
|
||||
query: r.URL.RawQuery,
|
||||
authorization: r.Header.Get("Authorization"),
|
||||
body: string(body),
|
||||
expect: r.Header.Get("Expect"),
|
||||
contentLength: r.ContentLength,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, response)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, recorded
|
||||
}
|
||||
|
||||
func hostedEnv(vars map[string]string) func(string) string {
|
||||
return func(key string) string { return vars[key] }
|
||||
}
|
||||
|
||||
func writeArchive(t *testing.T, name, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("writing archive: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestRunPush(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated,
|
||||
`{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`)
|
||||
archive := writeArchive(t, "node-20.1.0.tgz", "tarball")
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" {
|
||||
t.Errorf("request = %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
// sha256 of "tarball", so the proxy can check the upload.
|
||||
const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0"
|
||||
if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 {
|
||||
t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query)
|
||||
}
|
||||
if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) {
|
||||
t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength)
|
||||
}
|
||||
if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") {
|
||||
t.Errorf("output = %q, want filename and sha256", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushGenericNamesTheFile(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated,
|
||||
`{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`)
|
||||
archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball")
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
|
||||
t.Errorf("path = %q, want the file name appended", recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushErrors(t *testing.T) {
|
||||
srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`)
|
||||
archive := writeArchive(t, "serde-1.0.200.crate", "crate")
|
||||
|
||||
err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") {
|
||||
t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err)
|
||||
}
|
||||
|
||||
err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") {
|
||||
t.Errorf("rejected push error = %v, want the server's message", err)
|
||||
}
|
||||
|
||||
err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "usage") {
|
||||
t.Errorf("push with missing arguments error = %v, want usage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPushUsesProxyURLFromEnv(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`)
|
||||
archive := writeArchive(t, "request-2.88.3.tgz", "tarball")
|
||||
|
||||
err := runPush([]string{"npm", "request", "2.88.3", archive},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runPush: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/npm/request/2.88.3" {
|
||||
t.Errorf("path = %q", recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunList(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusOK,
|
||||
`{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`)
|
||||
var stdout bytes.Buffer
|
||||
|
||||
if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil {
|
||||
t.Fatalf("runList: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" {
|
||||
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
|
||||
}
|
||||
if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") {
|
||||
t.Errorf("output = %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDelete(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runDelete: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" {
|
||||
t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization)
|
||||
}
|
||||
|
||||
missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`)
|
||||
err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") {
|
||||
t.Errorf("delete of a missing version error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeleteOneFile(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("runDelete: %v", err)
|
||||
}
|
||||
if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
|
||||
t.Errorf("path = %q", recorded.path)
|
||||
}
|
||||
if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" {
|
||||
t.Errorf("output = %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
|
||||
|
||||
err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"},
|
||||
hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "only generic files") {
|
||||
t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err)
|
||||
}
|
||||
if recorded.method != "" {
|
||||
t.Errorf("refused delete sent %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHostedDispatchesSubcommands(t *testing.T) {
|
||||
srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`)
|
||||
|
||||
if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil {
|
||||
t.Fatalf("runHosted list: %v", err)
|
||||
}
|
||||
if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" {
|
||||
t.Errorf("request = %s %s", recorded.method, recorded.path)
|
||||
}
|
||||
|
||||
for _, args := range [][]string{nil, {"bogus"}} {
|
||||
err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client())
|
||||
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted <push|list|delete>") {
|
||||
t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) {
|
||||
err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient)
|
||||
if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") {
|
||||
t.Errorf("push usage error = %v, want it to name proxy hosted push", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -5,11 +5,6 @@
|
|||
// @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing.
|
||||
// @BasePath /
|
||||
//
|
||||
// @securityDefinitions.apikey BearerAuth
|
||||
// @in header
|
||||
// @name Authorization
|
||||
// @description "Bearer " followed by hosted.token, or projects.token for /api/projects
|
||||
//
|
||||
// The proxy caches package artifacts from upstream registries (npm, cargo, etc.)
|
||||
// providing faster, more reliable access for development teams.
|
||||
//
|
||||
|
|
@ -22,7 +17,6 @@
|
|||
// serve Start the proxy server (default if no command given)
|
||||
// stats Show cache statistics
|
||||
// mirror Pre-populate cache from PURLs, SBOMs, or registries
|
||||
// hosted Push, list or delete packages on a running proxy (needs hosted.enabled)
|
||||
//
|
||||
// Serve Flags:
|
||||
//
|
||||
|
|
@ -46,8 +40,6 @@
|
|||
// Log level: debug, info, warn, error (default "info")
|
||||
// -log-format string
|
||||
// Log format: text, json (default "text")
|
||||
// -access-log string
|
||||
// Path to the JSONL access log (disabled by default)
|
||||
//
|
||||
// Stats Flags:
|
||||
//
|
||||
|
|
@ -80,23 +72,14 @@
|
|||
// PROXY_DATABASE_URL - PostgreSQL connection URL
|
||||
// PROXY_LOG_LEVEL - Log level
|
||||
// PROXY_LOG_FORMAT - Log format
|
||||
// PROXY_ACCESS_LOG_PATH - JSONL access log path
|
||||
// PROXY_UPSTREAM_* - Upstream URLs and network access controls
|
||||
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Set false to allow Gradle PUT uploads (default true)
|
||||
// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL
|
||||
// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL
|
||||
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
|
||||
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
|
||||
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
|
||||
// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted
|
||||
// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting
|
||||
// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB")
|
||||
// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only
|
||||
// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only
|
||||
// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects)
|
||||
// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs
|
||||
// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d")
|
||||
// PROXY_URL - Proxy URL for proxy hosted
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
|
|
@ -121,17 +104,16 @@ import (
|
|||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/config"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/handler"
|
||||
"github.com/git-pkgs/proxy/internal/memlimit"
|
||||
"github.com/git-pkgs/proxy/internal/mirror"
|
||||
"github.com/git-pkgs/proxy/internal/server"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
const defaultTopN = 10
|
||||
|
|
@ -144,15 +126,6 @@ var (
|
|||
Commit = "unknown"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if Version != "dev" {
|
||||
return
|
||||
}
|
||||
if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" {
|
||||
Version = bi.Main.Version
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 {
|
||||
switch os.Args[1] {
|
||||
|
|
@ -168,10 +141,6 @@ func main() {
|
|||
os.Args = append(os.Args[:1], os.Args[2:]...)
|
||||
runMirror()
|
||||
return
|
||||
case "hosted":
|
||||
os.Args = append(os.Args[:1], os.Args[2:]...)
|
||||
runHostedCommand()
|
||||
return
|
||||
case "-version", "--version":
|
||||
fmt.Printf("proxy %s (%s)\n", Version, Commit)
|
||||
os.Exit(0)
|
||||
|
|
@ -194,7 +163,6 @@ Commands:
|
|||
serve Start the proxy server (default)
|
||||
stats Show cache statistics
|
||||
mirror Pre-populate cache from PURLs, SBOMs, or registries
|
||||
hosted Push, list or delete packages on a running proxy
|
||||
|
||||
Run 'proxy <command> -help' for more information on a command.
|
||||
|
||||
|
|
@ -216,7 +184,6 @@ func runServe() {
|
|||
databaseURL := fs.String("database-url", "", "PostgreSQL connection URL")
|
||||
logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error")
|
||||
logFormat := fs.String("log-format", "", "Log format: text, json")
|
||||
accessLogPath := fs.String("access-log", "", "Path to the JSONL access log")
|
||||
version := fs.Bool("version", false, "Print version and exit")
|
||||
|
||||
fs.Usage = func() {
|
||||
|
|
@ -234,36 +201,9 @@ func runServe() {
|
|||
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_ACCESS_LOG_PATH JSONL access log path\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS Comma-separated private upstream hosts\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_LOOPBACK Permit loopback upstreams and redirects\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NPM npm registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO Cargo index upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO_DOWNLOAD Cargo download upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GEM RubyGems upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO Go module proxy upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO_VULNDB Go vulnerability database URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX Hex repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX_API Hex API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PUB pub registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI PyPI index and API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI_DOWNLOAD PyPI download upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET NuGet API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET_SEARCH NuGet search upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_SWIFT Swift Package Registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_OCI_DEFAULT Default OCI registry upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_DEBIAN Debian repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_RPM RPM repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Set false to allow Gradle PUT uploads (default true)\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
|
||||
|
|
@ -316,9 +256,6 @@ func runServe() {
|
|||
if *logFormat != "" {
|
||||
cfg.Log.Format = *logFormat
|
||||
}
|
||||
if *accessLogPath != "" {
|
||||
cfg.AccessLog.Path = *accessLogPath
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if err := cfg.Validate(); err != nil {
|
||||
|
|
@ -328,13 +265,9 @@ func runServe() {
|
|||
|
||||
// Setup logger
|
||||
logger := setupLogger(cfg.Log.Level, cfg.Log.Format)
|
||||
memlimit.Apply(logger)
|
||||
|
||||
// Create and start server
|
||||
srv, err := server.New(cfg, logger, server.BuildInfo{
|
||||
Version: Version,
|
||||
Commit: Commit,
|
||||
})
|
||||
srv, err := server.New(cfg, logger)
|
||||
if err != nil {
|
||||
logger.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -443,8 +376,6 @@ func runMirror() {
|
|||
sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
|
||||
concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
|
||||
dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
|
||||
platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
|
||||
latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version")
|
||||
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
|
||||
|
|
@ -452,36 +383,21 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, "Examples:\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
|
||||
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n")
|
||||
fmt.Fprintf(os.Stderr, "Flags:\n")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
_ = fs.Parse(os.Args[1:])
|
||||
purls := fs.Args()
|
||||
var platformList []string
|
||||
if *platforms != "" {
|
||||
platformList = []string{*platforms}
|
||||
}
|
||||
if _, err := mirror.ParsePlatforms(platformList); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Determine source
|
||||
var source mirror.Source
|
||||
switch {
|
||||
case *sbomPath != "":
|
||||
data, err := os.ReadFile(*sbomPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList}
|
||||
source = &mirror.SBOMSource{Path: *sbomPath}
|
||||
case len(purls) > 0:
|
||||
source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList}
|
||||
source = &mirror.PURLSource{PURLs: purls}
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
|
||||
fs.Usage()
|
||||
|
|
@ -495,7 +411,6 @@ func runMirror() {
|
|||
os.Exit(1)
|
||||
}
|
||||
cfg.LoadFromEnv()
|
||||
printEnvWarnings(cfg)
|
||||
|
||||
if *storageURL != "" {
|
||||
cfg.Storage.URL = *storageURL
|
||||
|
|
@ -514,10 +429,6 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if !cfg.Storage.CacheArtifacts {
|
||||
fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
logger := setupLogger("info", "text")
|
||||
|
||||
|
|
@ -553,27 +464,15 @@ func runMirror() {
|
|||
os.Exit(1) //nolint:gocritic // db closed above
|
||||
}
|
||||
|
||||
// Build proxy with the same upstream clients and authentication as serve
|
||||
proxy, images := server.NewMirrorProxy(cfg, db, store, logger)
|
||||
// Build proxy (reuses same pipeline as serve)
|
||||
fetcher := fetch.NewFetcher()
|
||||
resolver := fetch.NewResolver()
|
||||
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
|
||||
proxy.CacheMetadata = true // mirror always caches metadata
|
||||
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
||||
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
|
||||
proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize()
|
||||
// As serve does, resolve Swift source repositories in the denylist through
|
||||
// the upstream, and refuse to run when one can't be.
|
||||
if err := server.ConfigureDenylist(cfg, proxy); err != nil {
|
||||
_ = db.Close()
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1) //nolint:gocritic // db closed above
|
||||
}
|
||||
|
||||
m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency)
|
||||
switch s := source.(type) {
|
||||
case *mirror.PURLSource:
|
||||
s.Versions = m.Versions
|
||||
case *mirror.SBOMSource:
|
||||
s.Versions = m.Versions
|
||||
}
|
||||
m := mirror.New(proxy, db, store, logger, *concurrency)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
|
|
@ -596,9 +495,7 @@ func runMirror() {
|
|||
return
|
||||
}
|
||||
|
||||
closeGit := useCommandGit(cfg, proxy, m, logger)
|
||||
progress, err := m.Run(ctx, source)
|
||||
closeGit()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -613,75 +510,6 @@ func runMirror() {
|
|||
fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// A partial mirror has to be visible in the exit status, or a CI step
|
||||
// that prewarms the cache passes even when every download failed.
|
||||
if code := mirrorExitCode(progress); code != 0 {
|
||||
_ = db.Close()
|
||||
os.Exit(code) //nolint:gocritic // db closed above
|
||||
}
|
||||
}
|
||||
|
||||
// newCommandMirror builds the mirror proxy mirror runs, which fetches and
|
||||
// lists versions from the configured upstreams.
|
||||
func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler,
|
||||
db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror {
|
||||
m := mirror.New(proxy, db, store, logger, workers)
|
||||
m.Images = images
|
||||
m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI)
|
||||
m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
|
||||
server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient)
|
||||
if cfg.Scanning.Enabled {
|
||||
refuseUnscannedMirroring(m)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
|
||||
// when upstream.git is configured, and returns the function that closes it.
|
||||
// Where it can't be opened, such as while the running server holds
|
||||
// git.cache_dir, those items fail with the reason and the rest are mirrored.
|
||||
func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
|
||||
git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
|
||||
if err != nil {
|
||||
m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
|
||||
return func() {}
|
||||
}
|
||||
if git == nil {
|
||||
return func() {}
|
||||
}
|
||||
m.Git = git
|
||||
return func() {
|
||||
if err := closeGit(); err != nil {
|
||||
logger.Warn("closing the git cache", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorExitCode returns the exit status for a finished mirror run: 1 when any
|
||||
// package version failed to mirror, 0 otherwise.
|
||||
func mirrorExitCode(progress *mirror.Progress) int {
|
||||
if progress != nil && progress.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Reasons the mirror command gives for refusing items while scanning is
|
||||
// enabled.
|
||||
const (
|
||||
mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
|
||||
mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
|
||||
)
|
||||
|
||||
// refuseUnscannedMirroring makes m fail every item instead of caching it.
|
||||
// Scanners fetch the staged bytes from the running server, which this command
|
||||
// does not start, so anything it cached would skip the pre-cache scan and
|
||||
// later be served from the cache unscanned. The mirror API's jobs run inside
|
||||
// the server with its scanners and are unaffected.
|
||||
func refuseUnscannedMirroring(m *mirror.Mirror) {
|
||||
m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason}
|
||||
m.RefusePackages = mirrorScanningPackagesReason
|
||||
}
|
||||
|
||||
func printStats(db *database.DB, popular, recent int, asJSON bool) error {
|
||||
|
|
@ -826,14 +654,6 @@ func formatSize(bytes int64) string {
|
|||
}
|
||||
}
|
||||
|
||||
// printEnvWarnings reports environment values LoadFromEnv ignored. serve
|
||||
// logs them from server.Start; mirror never gets there.
|
||||
func printEnvWarnings(cfg *config.Config) {
|
||||
for _, warning := range cfg.EnvWarnings() {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s\n", warning)
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(path string) (*config.Config, error) {
|
||||
if path != "" {
|
||||
return config.Load(path)
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestServeHelpProcess$")
|
||||
cmd.Env = append(os.Environ(), "PROXY_TEST_SERVE_HELP=1")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("serve help failed: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
variables := []string{
|
||||
"PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS",
|
||||
"PROXY_UPSTREAM_ALLOW_LOOPBACK",
|
||||
"PROXY_UPSTREAM_NPM",
|
||||
"PROXY_UPSTREAM_CARGO",
|
||||
"PROXY_UPSTREAM_CARGO_DOWNLOAD",
|
||||
"PROXY_UPSTREAM_GEM",
|
||||
"PROXY_UPSTREAM_GO",
|
||||
"PROXY_UPSTREAM_HEX",
|
||||
"PROXY_UPSTREAM_HEX_API",
|
||||
"PROXY_UPSTREAM_PUB",
|
||||
"PROXY_UPSTREAM_PYPI",
|
||||
"PROXY_UPSTREAM_PYPI_DOWNLOAD",
|
||||
"PROXY_UPSTREAM_MAVEN",
|
||||
"PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL",
|
||||
"PROXY_UPSTREAM_NUGET",
|
||||
"PROXY_UPSTREAM_NUGET_SEARCH",
|
||||
"PROXY_UPSTREAM_COMPOSER",
|
||||
"PROXY_UPSTREAM_COMPOSER_REPOSITORY",
|
||||
"PROXY_UPSTREAM_CONAN",
|
||||
"PROXY_UPSTREAM_CHEF",
|
||||
"PROXY_UPSTREAM_CONDA",
|
||||
"PROXY_UPSTREAM_CRAN",
|
||||
"PROXY_UPSTREAM_JULIA",
|
||||
"PROXY_UPSTREAM_OCI_DEFAULT",
|
||||
"PROXY_UPSTREAM_DEBIAN",
|
||||
"PROXY_UPSTREAM_RPM",
|
||||
}
|
||||
for _, variable := range variables {
|
||||
if !strings.Contains(string(output), variable) {
|
||||
t.Errorf("serve help omitted %s", variable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeHelpProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_SERVE_HELP") != "1" {
|
||||
return
|
||||
}
|
||||
os.Args = []string{"proxy", "serve", "-help"}
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMirrorLogsEnvWarnings(t *testing.T) {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$")
|
||||
// The negative size stops mirror before it opens any storage.
|
||||
cmd.Env = append(os.Environ(),
|
||||
"PROXY_TEST_MIRROR_ENV_WARNINGS=1",
|
||||
"PROXY_CACHE_METADATA=maybe",
|
||||
"PROXY_STORAGE_MAX_SIZE=-1")
|
||||
output, _ := cmd.CombinedOutput()
|
||||
if !strings.Contains(string(output), "PROXY_CACHE_METADATA") {
|
||||
t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorEnvWarningsProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" {
|
||||
return
|
||||
}
|
||||
os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"}
|
||||
main()
|
||||
}
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/gitcache"
|
||||
)
|
||||
|
||||
// gitMirrorUpstream serves team/repo, with one commit on main, from git
|
||||
// http-backend, and returns the server and the commit.
|
||||
func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
|
||||
t.Helper()
|
||||
git := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = []string{
|
||||
"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
|
||||
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
backend := ""
|
||||
for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
|
||||
if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
|
||||
backend = path
|
||||
}
|
||||
}
|
||||
if backend == "" {
|
||||
t.Skip("git http-backend is not installed")
|
||||
}
|
||||
root := t.TempDir()
|
||||
bare := filepath.Join(root, "team", "repo.git")
|
||||
git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
|
||||
work := t.TempDir()
|
||||
git("", "clone", "--quiet", bare, work)
|
||||
git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
|
||||
git(work, "push", "--quiet", "origin", "HEAD:main")
|
||||
head := git(work, "rev-parse", "HEAD")
|
||||
|
||||
server := httptest.NewServer(&cgi.Handler{
|
||||
Path: backend,
|
||||
Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
|
||||
})
|
||||
t.Cleanup(server.Close)
|
||||
return server, head
|
||||
}
|
||||
|
||||
func fileExistsForTest(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeGitMirrorConfig writes a config with upstream.git "up" at
|
||||
// upstreamURL and the git cache, storage and database under dir.
|
||||
func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"upstream:\n" +
|
||||
" allow_loopback: true\n" +
|
||||
" git:\n" +
|
||||
" up: " + upstreamURL + "\n" +
|
||||
"git:\n" +
|
||||
" cache_dir: " + filepath.Join(dir, "git") + "\n"
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestMirrorCommandCachesGitCommit(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 downloaded") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
|
||||
if len(cached) != 1 {
|
||||
t.Errorf("cached repositories %v, want team/repo.git", cached)
|
||||
}
|
||||
|
||||
output = runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 skipped") {
|
||||
t.Errorf("second run output:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
|
||||
// mirror API, while the running server holds git.cache_dir.
|
||||
func TestMirrorCommandGitCacheInUse(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
cache, err := gitcache.New(gitcache.Options{
|
||||
CacheDir: filepath.Join(dir, "git"),
|
||||
RefreshWait: time.Second,
|
||||
Client: &http.Client{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = cache.Close() }()
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, purl)
|
||||
for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
)
|
||||
|
||||
const swiftWidgetArchive = "acme widget archive"
|
||||
|
||||
// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package
|
||||
// registry does, with https://github.com/acme/widget as its source
|
||||
// repository, and counts archive downloads and identifier lookups. The
|
||||
// repository's identifiers are ids, or only acme.widget.
|
||||
func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) {
|
||||
t.Helper()
|
||||
if len(ids) == 0 {
|
||||
ids = []string{"acme.widget"}
|
||||
}
|
||||
identifiers, err := json.Marshal(map[string][]string{"identifiers": ids})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloads, lookups = new(atomic.Int64), new(atomic.Int64)
|
||||
sum := sha256.Sum256([]byte(swiftWidgetArchive))
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Version", "1")
|
||||
switch r.URL.Path {
|
||||
case "/identifiers":
|
||||
lookups.Add(1)
|
||||
if r.URL.Query().Get("url") != "https://github.com/acme/widget" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(identifiers)
|
||||
case "/acme/widget":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`)
|
||||
case "/acme/widget/1.0.0/Package.swift":
|
||||
w.Header().Set("Content-Type", "text/x-swift")
|
||||
_, _ = io.WriteString(w, "// swift-tools-version:5.9\n")
|
||||
case "/acme/widget/1.0.0":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:]))
|
||||
case "/acme/widget/1.0.0.zip":
|
||||
downloads.Add(1)
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = io.WriteString(w, swiftWidgetArchive)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, downloads, lookups
|
||||
}
|
||||
|
||||
// writeSwiftMirrorConfig writes a config using upstream as the Swift
|
||||
// registry, denying the denied package PURLs.
|
||||
func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"upstream:\n" +
|
||||
" swift: " + upstream + "\n" +
|
||||
" allow_loopback: true\n"
|
||||
if len(denied) > 0 {
|
||||
cfg += "denylist:\n packages:\n"
|
||||
for _, p := range denied {
|
||||
cfg += " - " + p + "\n"
|
||||
}
|
||||
}
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func cachedArtifacts(t *testing.T, dir string) int64 {
|
||||
t.Helper()
|
||||
db, err := database.Open(filepath.Join(dir, "proxy.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("opening database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
stats, err := db.GetCacheStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetCacheStats: %v", err)
|
||||
}
|
||||
return stats.TotalArtifacts
|
||||
}
|
||||
|
||||
// Without a denylist the fake registry mirrors the release, so the tests
|
||||
// below fail for the denylist and not for the setup.
|
||||
func TestMirrorSwiftRelease(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL)
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "1 downloaded") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 1 {
|
||||
t.Errorf("archive downloads = %d, want 1", n)
|
||||
}
|
||||
if n := cachedArtifacts(t, dir); n != 1 {
|
||||
t.Errorf("cached artifacts = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A denylist entry naming the package by its source repository blocks the
|
||||
// mirror download of the registry identity it resolves to.
|
||||
func TestMirrorDenylistSwiftSourceEntry(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
if n := cachedArtifacts(t, dir); n != 0 {
|
||||
t.Errorf("cached artifacts = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The registry may list several identities for one repository. A source
|
||||
// entry denies all of them, not only the first.
|
||||
func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) {
|
||||
upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget")
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A source repository entry the registry can't resolve stops the mirror
|
||||
// before it downloads anything, as it stops serve: left out, it would let
|
||||
// the version through.
|
||||
func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) {
|
||||
upstream, downloads, lookups := swiftMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0")
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
|
||||
|
||||
if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") {
|
||||
t.Errorf("output does not report the unresolved entry:\n%s", output)
|
||||
}
|
||||
if strings.Contains(output, "Mirror complete") {
|
||||
t.Errorf("mirror ran:\n%s", output)
|
||||
}
|
||||
if lookups.Load() == 0 {
|
||||
t.Error("the entry was not looked up")
|
||||
}
|
||||
if n := downloads.Load(); n != 0 {
|
||||
t.Errorf("archive downloads = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,214 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/mirror"
|
||||
)
|
||||
|
||||
// mirrorTestEnv carries the mirror command's arguments into the helper
|
||||
// process, one per line.
|
||||
const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS"
|
||||
|
||||
// writeScanningMirrorConfig writes a config with scanning enabled and storage
|
||||
// and database under dir. The scanner address is never reached.
|
||||
func writeScanningMirrorConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"scanning:\n" +
|
||||
" enabled: true\n" +
|
||||
" signing_key: test-signing-key\n" +
|
||||
" scanners:\n" +
|
||||
" - name: test\n" +
|
||||
" url: http://127.0.0.1:1/scan\n"
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// runMirrorProcess runs proxy mirror with args in a helper process and
|
||||
// checks that it exits with wantCode.
|
||||
func runMirrorProcess(t *testing.T, wantCode int, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$")
|
||||
cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n"))
|
||||
output, err := cmd.CombinedOutput()
|
||||
|
||||
code := 0
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
code = exitErr.ExitCode()
|
||||
} else if err != nil {
|
||||
t.Fatalf("proxy mirror %v: %v\n%s", args, err, output)
|
||||
}
|
||||
if code != wantCode {
|
||||
t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output)
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
// With scanning enabled, proxy mirror refuses packages rather than caching
|
||||
// them unscanned: scanners fetch staged bytes from the running server, which
|
||||
// the command does not start.
|
||||
// A refused package counts as failed, so the command exits 1.
|
||||
func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeScanningMirrorConfig(t, dir)
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21")
|
||||
|
||||
for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
|
||||
db, err := database.Open(filepath.Join(dir, "proxy.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("opening database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
stats, err := db.GetCacheStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetCacheStats: %v", err)
|
||||
}
|
||||
if stats.TotalArtifacts != 0 {
|
||||
t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts)
|
||||
}
|
||||
|
||||
var stored []string
|
||||
_ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
stored = append(stored, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(stored) != 0 {
|
||||
t.Errorf("files written to storage: %v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run caches nothing, so it still lists what would be mirrored.
|
||||
func TestMirrorCommandDryRunWhileScanning(t *testing.T) {
|
||||
configPath := writeScanningMirrorConfig(t, t.TempDir())
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21")
|
||||
|
||||
if !strings.Contains(output, "Would mirror 1 package versions") {
|
||||
t.Errorf("dry run output:\n%s", output)
|
||||
}
|
||||
if strings.Contains(output, mirrorScanningPackagesReason) {
|
||||
t.Errorf("dry run reported the refusal:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorCommandSBOMComponentVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeScanningMirrorConfig(t, dir)
|
||||
sbomPath := filepath.Join(dir, "bom.json")
|
||||
data := []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`)
|
||||
if err := os.WriteFile(sbomPath, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "-sbom", sbomPath)
|
||||
for _, want := range []string{"Would mirror 1 package versions", "pkg:docker/ghcr.io/team/app@1.0"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("dry run missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefuseUnscannedMirroring(t *testing.T) {
|
||||
m := mirror.New(nil, nil, nil, nil, 1)
|
||||
refuseUnscannedMirroring(m)
|
||||
|
||||
if m.RefusePackages != mirrorScanningPackagesReason {
|
||||
t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason)
|
||||
}
|
||||
images, ok := m.Images.(mirror.UnavailableImages)
|
||||
if !ok || images.Reason != mirrorScanningImagesReason {
|
||||
t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorHelperProcess(*testing.T) {
|
||||
args := os.Getenv(mirrorTestEnv)
|
||||
if args == "" {
|
||||
return
|
||||
}
|
||||
os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...)
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMirrorExitCode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
progress *mirror.Progress
|
||||
want int
|
||||
}{
|
||||
{"nil progress", nil, 0},
|
||||
{"everything mirrored", &mirror.Progress{Total: 2, Completed: 2}, 0},
|
||||
{"everything cached", &mirror.Progress{Total: 2, Skipped: 2}, 0},
|
||||
{"one failure", &mirror.Progress{Total: 2, Completed: 1, Failed: 1}, 1},
|
||||
{"all failed", &mirror.Progress{Total: 3, Failed: 3}, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := mirrorExitCode(tt.progress); got != tt.want {
|
||||
t.Errorf("mirrorExitCode() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// proxy mirror used to exit 0 after printing "N failed", so a CI step that
|
||||
// prewarms the cache passed even when every download failed (#500). An
|
||||
// ecosystem the resolver rejects fails without touching the network.
|
||||
func TestMirrorExitsNonZeroWhenDownloadsFail(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorFailureProcess$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"PROXY_TEST_MIRROR_FAILURE=1",
|
||||
"PROXY_TEST_MIRROR_DIR="+dir,
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) {
|
||||
t.Fatalf("mirror with a failed download exited with %v, want status 1\n%s", err, output)
|
||||
}
|
||||
if code := exitErr.ExitCode(); code != 1 {
|
||||
t.Fatalf("exit status = %d, want 1\n%s", code, output)
|
||||
}
|
||||
if !strings.Contains(string(output), "1 failed") {
|
||||
t.Errorf("output does not report the failure:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(string(output), "bogus/foo@1.0.0") {
|
||||
t.Errorf("output does not list the failed package:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorFailureProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_MIRROR_FAILURE") != "1" {
|
||||
return
|
||||
}
|
||||
dir := os.Getenv("PROXY_TEST_MIRROR_DIR")
|
||||
os.Args = []string{
|
||||
"proxy", "mirror",
|
||||
"--storage-url", "file://" + filepath.Join(dir, "store"),
|
||||
"--database-path", filepath.Join(dir, "cache.db"),
|
||||
"pkg:bogus/foo@1.0.0",
|
||||
}
|
||||
main()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue